{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [],
   "source": [
    "## US Public Debt and Safe Asset Market Power\n",
    "## Jason Choi, Rishabh Kirpalani, and Diego Perez\n",
    "## Sept 1, 2023\n",
    "## Python Code by Duong Dang\n",
    "\n",
    "## Replication of Empirical, including Appendix.\n",
    " \n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from statsmodels.api import add_constant\n",
    "from linearmodels.iv import IV2SLS\n",
    "from statsmodels.iolib.summary2 import summary_col\n",
    "from linearmodels.panel import compare\n",
    "from stargazer.stargazer import Stargazer\n",
    "from scipy.stats import norm\n",
    "from IPython.core.display import HTML\n",
    "from statsmodels.sandbox.regression.gmm import IV2SLS as smIV2SLS\n",
    "import statsmodels.api as sm\n",
    "from scipy.linalg import fractional_matrix_power as fmp\n",
    "import latextable, texttable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constructing dependency ratio\n",
    "\n",
    "temp = pd.read_csv(r'population.txt', index_col = 0).T\n",
    "\n",
    "temp.columns = ['a_pop', 'dep']\n",
    "\n",
    "temp['M'] = temp.index.str[:3]\n",
    "temp['Y'] = temp.index.str[4:]\n",
    "temp['YM'] = temp.Y + temp.M\n",
    "\n",
    "temp.index = pd.to_datetime(temp.YM, format = '%Y%b')\n",
    "\n",
    "temp.drop(columns = ['M', 'Y', 'YM'], inplace = True)\n",
    "\n",
    "temp['a_pop'] = temp.a_pop.astype(str)\n",
    "temp['dep'] = temp.dep.astype(str)\n",
    "\n",
    "temp['a_pop'] = temp.a_pop.str.split(pat = '(').str[0]\n",
    "temp['dep'] = temp.dep.str.split(pat = '(').str[0]\n",
    "\n",
    "for col in temp.columns:\n",
    "    temp[col] = pd.to_numeric(temp[col], errors = 'coerce')\n",
    "\n",
    "temp.dropna(inplace = True)\n",
    "\n",
    "temp['year'] = temp.index.year\n",
    "temp['quarter'] = temp.index.quarter\n",
    "\n",
    "temp2 = temp.groupby(['year', 'quarter']).mean()\n",
    "\n",
    "temp2.reset_index(inplace = True)\n",
    "\n",
    "temp2['date'] = pd.to_datetime(temp2.year.astype(str) + '-Q' + temp2.quarter.astype(str))\n",
    "\n",
    "temp2['dep_rat'] = temp2.dep / (temp2.a_pop - temp2.dep)\n",
    "\n",
    "sparam = 1600\n",
    "\n",
    "cycle, trend = sm.tsa.filters.hpfilter(temp2.a_pop, sparam)\n",
    "temp2['a_pop_s'] = trend\n",
    "\n",
    "cycle, trend = sm.tsa.filters.hpfilter(temp2.dep, sparam)\n",
    "temp2['dep_s'] = trend\n",
    "\n",
    "temp2['dep_rat_s'] = sm.tsa.filters.hpfilter(temp2.dep_rat, sparam)[1]\n",
    "\n",
    "df2 = temp2[['date', 'dep_rat_s']].copy()\n",
    "\n",
    "df2.columns = ['date', 'dep_rat']\n",
    "\n",
    "df1 = pd.read_csv('dep_rat_a.csv')[['year', 'oa_dep']]\n",
    "\n",
    "df1.rename(columns = {'oa_dep': 'dep_rat'}, inplace = True)\n",
    "\n",
    "factor = df2[df2.date == '1948-01-01'].dep_rat.values[0] / df1[df1.year == 1948].dep_rat.values[0]\n",
    "\n",
    "df1.dep_rat = df1.dep_rat * factor\n",
    "\n",
    "df1 = df1[df1.year <= 1948].copy()\n",
    "\n",
    "df1['date'] = pd.to_datetime(df1.year, format = '%Y')\n",
    "\n",
    "df1 = df1[['date', 'dep_rat']].copy()\n",
    "df1.columns = ['date', 'dep_rat']\n",
    "\n",
    "df2.date  = pd.to_datetime(df2.date)\n",
    "\n",
    "df1.set_index('date', inplace = True)\n",
    "\n",
    "df1 = df1.asfreq('QS')\n",
    "\n",
    "df1.reset_index(inplace = True)\n",
    "\n",
    "df1.dep_rat = df1.dep_rat.interpolate()\n",
    "\n",
    "df1 = df1[df1.date < '1948'].copy()\n",
    "\n",
    "df = pd.concat([df1, df2])\n",
    "\n",
    "df.set_index('date', inplace = True)\n",
    "\n",
    "df.reset_index(inplace = True)\n",
    "\n",
    "df['year'] = df.date.dt.year\n",
    "\n",
    "df['q'] = df.date.dt.quarter\n",
    "\n",
    "df.date = df.year + (df.q - 1) * 0.25\n",
    "\n",
    "df['DLdep_rat_a'] = np.log(df.dep_rat / df.dep_rat.shift())\n",
    "\n",
    "df.rename(columns = {'dep_rat':'age_dep'}, inplace = True)\n",
    "\n",
    "df[['date', 'age_dep']].to_csv('age_dep.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.1: Volatility measure construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/1804542289.py:13: FutureWarning: Series.dt.weekofyear and Series.dt.week have been deprecated. Please use Series.dt.isocalendar().week instead.\n",
      "  df1['w'] = df1.date.dt.week\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/1804542289.py:20: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df1 = df1a[df1.groupby(['y', 'w']).size() > 1].reset_index()\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/1804542289.py:71: FutureWarning: In a future version of pandas all arguments of DataFrame.drop_duplicates except for the argument 'subset' will be keyword-only.\n",
      "  df1.drop_duplicates('date' ,'last', inplace = True)\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/1804542289.py:83: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "  df['vix_hat'] = np.nan\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_stata('quarterly.dta')\n",
    "\n",
    "for series in ['msdluk']:\n",
    "    \n",
    "    df1 = pd.read_excel('ftse.xlsx', sheet_name = series)\n",
    "    df1.columns = ['date', series, 'n0', 'n1']\n",
    "    df1 = df1[['date', series]].copy()\n",
    "\n",
    "    df1['date'] = pd.to_datetime(df1.date)\n",
    "    df1['q'] = df1.date.dt.quarter\n",
    "    df1['y'] = df1.date.dt.year\n",
    "\n",
    "    df1['w'] = df1.date.dt.week\n",
    "\n",
    "    df1 = df1[df1[series].notna()].copy()\n",
    "\n",
    "    df1 = df1[df1.date >= '1972'].copy()\n",
    "\n",
    "    df1a = df1.set_index(['y', 'w'])\n",
    "    df1 = df1a[df1.groupby(['y', 'w']).size() > 1].reset_index()\n",
    "\n",
    "    df1a = df1.sort_values('date').groupby(['y', 'w'])[series].agg(['first', 'last']).reset_index()\n",
    "\n",
    "    df1a['ret'] = (df1a['last'] - df1a['first']) / df1a['first']\n",
    "\n",
    "    df1a['formatted_date'] = df1a.y * 1000 + df1a.w * 10 + 0\n",
    "    df1a['date'] = pd.to_datetime(df1a['formatted_date'], format='%Y%W%w')\n",
    "\n",
    "    df1a['q'] = df1a.date.dt.quarter\n",
    "\n",
    "    df1 = df1a.drop(df1a.iloc[[0, -1]].index)\n",
    "\n",
    "    tab = []\n",
    "    for year in range(df1.y.min() + 1, df1.y.max() + 1):\n",
    "        for quarter in range(1, 5):\n",
    "            if quarter == 1:\n",
    "                ind = ((df1.y == year) & (df1.q == 1)) | ((df1.y == year-1) & (df1.q == 4)) \\\n",
    "                    | ((df1.y == year-1) & (df1.q == 3)) | ((df1.y == year-1) & (df1.q == 2))\n",
    "            elif quarter == 2:\n",
    "                ind = ((df1.y == year) & (df1.q == 2)) | ((df1.y == year) & (df1.q == 1)) \\\n",
    "                    | ((df1.y == year-1) & (df1.q == 4)) | ((df1.y == year-1) & (df1.q == 3))\n",
    "            elif quarter == 3:\n",
    "                ind = ((df1.y == year) & (df1.q == 3)) | ((df1.y == year) & (df1.q == 2)) \\\n",
    "                    | ((df1.y == year) & (df1.q == 1)) | ((df1.y == year-1) & (df1.q == 4))\n",
    "            elif quarter == 4:\n",
    "                ind = ((df1.y == year) & (df1.q == 4)) | ((df1.y == year) & (df1.q == 3)) \\\n",
    "                    | ((df1.y == year) & (df1.q == 2)) | ((df1.y == year) & (df1.q == 1))\n",
    "            tab.append([year, quarter, df1.loc[ind].ret.std()])\n",
    "\n",
    "    df1 = pd.DataFrame(tab, columns = ['y', 'q', series + '_vol'])\n",
    "\n",
    "    df1['date'] = df1.y + (df1.q - 1) * 0.25\n",
    "    \n",
    "    df = df.merge(df1[['date', series + '_vol']], on = 'date', how = 'left')\n",
    "    \n",
    "df1 = pd.read_csv('MSPIUKM.csv')\n",
    "df1.columns = ['date', 'uks']\n",
    "\n",
    "df1['uks_L'] = df1.uks.shift()\n",
    "df1['ret'] = df1.uks / df1.uks_L - 1\n",
    "\n",
    "df1.date = pd.to_datetime(df1.date)\n",
    "\n",
    "df1['q'] = df1.date.dt.quarter\n",
    "df1['y'] = df1.date.dt.year\n",
    "\n",
    "df1['ukm_vol'] = df1.ret.rolling(12).std()\n",
    "\n",
    "df1['date'] = df1.y + (df1.q - 1) * 0.25\n",
    "\n",
    "df1.drop_duplicates('date' ,'last', inplace = True)\n",
    "\n",
    "df = df.merge(df1[['date', 'ukm' + '_vol']], on = 'date', how = 'left')\n",
    "    \n",
    "df1 = pd.read_csv('age_dep.csv', index_col = 0)\n",
    "df = df.merge(df1, on = 'date', how = 'left')\n",
    "df.DLdep_rat_new = np.log(df.age_dep / df.age_dep.shift())\n",
    "\n",
    "df1 = pd.read_excel('TVR.xlsx', sheet_name = 'Data_plots')\n",
    "\n",
    "df = df.merge(df1[['date', 'VIX', 'EPU']], on = 'date', how = 'left')\n",
    "\n",
    "df['vix_hat'] = np.nan\n",
    "\n",
    "dft = df[df.date >= 1926].copy()\n",
    "exog = dft[['vol']].copy()\n",
    "\n",
    "res = sm.OLS(dft.VIX, add_constant(exog), missing = 'drop').fit()\n",
    "\n",
    "df.loc[df[df.date >= 1926].index, 'vix_hat'] = add_constant(exog) @ res.params\n",
    "df.loc[df[df.VIX.notna()].index, 'vix_hat'] = df.loc[df[df.VIX.notna()].index, 'VIX']\n",
    "\n",
    "proj2 = res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for series in ['msdluk_vol']:\n",
    "    dft = df[df.date >= 1926].copy()\n",
    "    exog = (dft[['ukm_vol']]).copy()\n",
    "\n",
    "    res = res = sm.OLS((dft[series]), add_constant(exog), missing = 'drop').fit()\n",
    "\n",
    "    df[series + '_hat'] = np.nan\n",
    "    \n",
    "    df.loc[(add_constant(exog) @ res.params).index, series + '_hat'] = (add_constant(exog) @ res.params)\n",
    "    df.loc[df[df[series].notna()].index, series + '_hat'] = df.loc[df[df[series].notna()].index, series]\n",
    "    \n",
    "    proj1 = res\n",
    "\n",
    "sc = summary_col([proj1, proj2], stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['ukm_vol', 'vol', 'const'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 547,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = sc.tables[0].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'MSCI UK Volatility', 'VIX']\n",
    "tab.iloc[:, 0] = ['UK Share Price Volatility', '', 'S\\\\&P500 Volatility', '', 'Const', '', 'Observations', 'R-squared']\n",
    "tab.iloc[-2, [1, 2]] = [int(proj1.nobs), int(proj2.nobs)]\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "tab.iloc[:, 2] = tab.iloc[:, 2].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\"])\n",
    "table.set_header_align(['c', 'c', 'c'])\n",
    "table.set_cols_dtype(['t', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "string = latextable.draw_latex(table, use_booktabs = True)\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb1.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.2: Foreign volatility residual projection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.spread_S60_L40.notna() & (df.date < 2020)].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']])\n",
    "res = sm.OLS(dft[var], XX).fit()\n",
    "\n",
    "sc = summary_col([res], stars = True, float_format = '%.4f',\n",
    "                 regressor_order = ['vix_hat_ma', 'rgdp_grate', 'const'])\n",
    "\n",
    "tab = sc.tables[0].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'UK Volatility']\n",
    "tab.iloc[:, 0] = ['US Volatility', '', 'Real GDP growth', '', 'Const', '', 'Observations', 'R-squared']\n",
    "tab.iloc[-2, 1] = int(res.nobs)\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "tab = tab.iloc[:-1, :]\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\"])\n",
    "table.set_header_align(['c', 'c'])\n",
    "table.set_cols_dtype(['t', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "string = latextable.draw_latex(table, use_booktabs = True)\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb1_resid.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/1328079892.py:26: FutureWarning: Passing a set as an indexer is deprecated and will raise in a future version. Use a list instead.\n",
      "  df_x.loc[df_x[df_x['row'].isna()].index, 'row'] = df_x1.loc[set(df_x[df_x['row'].isna()].index) & set(df_x1.index), 'row_tic'] \\\n"
     ]
    }
   ],
   "source": [
    "df_x = pd.read_csv('fof_l210b_q.csv')\n",
    "df_x.columns = ['date', 'private_pension', 'fed_retirement', 'state_retirement', 'money_market', 'mutual',\n",
    "              'close_ended', 'etf', 'gov_sponsored', 'abs', 'brokers', 'holding_co', 'row']\n",
    "df_x['date'] = pd.to_datetime(df_x.date)\n",
    "df_x['date'] = df_x.date.dt.year + (df_x.date.dt.quarter - 1) / 4\n",
    "df_x = df_x[['date', 'row']].copy()\n",
    "\n",
    "df_x = df_x.merge(df[['date', 'lprivatepar', 'ngdp']], on = 'date', how = 'right')\n",
    "\n",
    "df_x1 = pd.read_csv('tic.csv')[['Date', 'Amount']]\n",
    "df_x1.columns = ['date', 'row_tic']\n",
    "\n",
    "df_x1.drop(df_x1[df_x1.row_tic.isna()].index, inplace = True)\n",
    "\n",
    "df_x1.date = pd.to_datetime(df_x1.date, format = '%d-%b-%y')\n",
    "\n",
    "df_x1.loc[df_x1['date'].dt.year >= 2000, 'date'] -= pd.DateOffset(years=100)\n",
    "\n",
    "df_x1.date = df_x1.date.dt.year + (df_x1.date.dt.quarter - 1) / 4 + 0.25\n",
    "\n",
    "df_x1.row_tic *= 1000\n",
    "\n",
    "df_x.set_index('date', inplace = True)\n",
    "df_x1.set_index('date', inplace = True)\n",
    "\n",
    "df_x.loc[df_x[df_x['row'].isna()].index, 'row'] = df_x1.loc[set(df_x[df_x['row'].isna()].index) & set(df_x1.index), 'row_tic'] \\\n",
    "                                                * (df_x[df_x.index == 1952].row/df_x1[df_x1.index == 1952].row_tic).values\n",
    "\n",
    "df_x.reset_index(inplace = True)\n",
    "\n",
    "for i in ['row']:\n",
    "    df_x.loc[(df_x.date >= 1940) & (df_x.date <=1952), i] = df_x.loc[(df_x.date >= 1940)\n",
    "                                                                      & (df_x.date <=1952)][i].interpolate()\n",
    "\n",
    "df_x['ln_row_gdp'] = np.log(df_x['row'] / df_x['ngdp'] / 1000)\n",
    "df_x['row_share'] = np.exp(df_x.ln_row_gdp - df_x.lprivatepar)\n",
    "df_x.sort_values('date', inplace = True)\n",
    "\n",
    "df = df.merge(df_x[['date', 'row_share', 'ln_row_gdp']], on = 'date', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns = 'lexternal', inplace = True)\n",
    "df['lexternal'] = df['ln_row_gdp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 551,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfu = pd.read_csv(r'uk/uk_gdp.csv', skiprows = 82, names = ['date', 'gdp'])\n",
    "\n",
    "dfu['q'] = dfu.date.str[-1].astype(int)\n",
    "dfu['y'] = dfu.date.str[:4].astype(int)\n",
    "\n",
    "df1 = pd.read_csv(r'uk/uk_debt_m.csv')\n",
    "\n",
    "df1['date'] = pd.to_datetime(df1.year.astype(int).astype(str) + '-' + df1.month.astype(int).astype(str) + '-1')\n",
    "\n",
    "df1['q'] = df1.date.dt.quarter\n",
    "df1['y'] = df1.date.dt.year\n",
    "\n",
    "df1 = df1.groupby(['y', 'q']).debt_market.mean().reset_index()\n",
    "\n",
    "df1 = df1[df1.y >= 1730].copy()\n",
    "\n",
    "dfu = dfu.merge(df1[['y', 'q', 'debt_market']], on = ['y', 'q'], how = 'outer')\n",
    "\n",
    "df1 = pd.read_excel(r'uk/uk_gdp2.xlsx')\n",
    "df1['gdp_y'] = df1['gdp'] * 0.75 + df1['gdp'].shift() * 0.25\n",
    "df1.loc[df1.date < 1854, 'gdp_y'] = df1.loc[df1.date < 1854, 'gdp']\n",
    "\n",
    "df1['y'] = df1['date']\n",
    "df1['q'] = 1\n",
    "\n",
    "dfu = dfu.merge(df1[['y', 'q', 'gdp_y']], on = ['y', 'q'], how = 'outer')\n",
    "\n",
    "dfu.sort_values(['y', 'q'], inplace = True)\n",
    "\n",
    "dfu.date = dfu.y + (dfu.q - 1) / 4\n",
    "\n",
    "dfu['gdp'] = dfu.gdp * 4\n",
    "\n",
    "dfu['gdp_i'] = dfu.gdp\n",
    "dfu.loc[dfu.gdp_i.isna(), 'gdp_i'] = dfu.loc[dfu.gdp_i.isna(), 'gdp_y']\n",
    "dfu.gdp_i = dfu.gdp_i.interpolate()\n",
    "\n",
    "dfu['ldebtuk2'] = np.log(dfu.debt_market / dfu.gdp_i) - np.log(100)\n",
    "\n",
    "df = df.merge(dfu[['date', 'ldebtuk2', 'debt_market']], on = 'date', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = pd.read_csv('ba_cd.csv')\n",
    "\n",
    "ss.columns = ['date', 'ba_h', 'ba', 'cd']\n",
    "\n",
    "ss.date = pd.to_datetime(ss.date)\n",
    "\n",
    "ss.date = ss.date.dt.year + (ss.date.dt.quarter - 1)/4\n",
    "\n",
    "ss.ba_h = pd.to_numeric(ss.ba_h, errors = 'coerce')\n",
    "ss.cd = pd.to_numeric(ss.cd, errors = 'coerce')\n",
    "ss.ba = pd.to_numeric(ss.ba, errors = 'coerce')\n",
    "\n",
    "ss['safe_i'] = ss.cd\n",
    "\n",
    "df = df.merge(ss[['date', 'safe_i']], how = 'left', on = 'date')\n",
    "\n",
    "df['yield_short'] = df.cp - df.spread_short\n",
    "\n",
    "df['spread_sa'] = df.safe_i - df.yield_short"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 553,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/1033151588.py:17: UserWarning: You are merging on int and float columns where the float values are not equal to their int representation.\n",
      "  dfd = dfd.merge(dfb, on = 'date', how = 'outer')\n"
     ]
    }
   ],
   "source": [
    "dfd = pd.read_csv(r'SAVINGSL.csv')\n",
    "dfd = dfd.merge(pd.read_csv(r'STDSL.csv'), on = 'DATE')\n",
    "dfd = dfd.merge(pd.read_csv(r'TCDSL.csv'), on = 'DATE')\n",
    "\n",
    "dfd.columns = ['date', 'savings', 'time', 'demand']\n",
    "dfd.date = pd.to_datetime(dfd.date, format = \"%Y-%m-%d\")\n",
    "\n",
    "dfd['y'] = dfd.date.dt.year\n",
    "dfd['q'] = dfd.date.dt.quarter\n",
    "dfd = dfd.groupby(['y', 'q'])[['savings', 'time', 'demand']].mean().reset_index()\n",
    "dfd['date'] = dfd.y + (dfd.q - 1) /4\n",
    "\n",
    "dfb = pd.read_csv(r'bank-data.csv')[['date', 'savings', 'time', 'demand']]\n",
    "for i in ['savings', 'time', 'demand']:\n",
    "    dfb[i] /= 1e6\n",
    "\n",
    "dfd = dfd.merge(dfb, on = 'date', how = 'outer')\n",
    "\n",
    "dfd.sort_values('date', inplace = True)\n",
    "\n",
    "dfd['tol_dep'] = dfd.savings_x + dfd.time_x + dfd.demand_x\n",
    "\n",
    "dfd['tol2'] = dfd.savings_y + dfd.time_y + dfd.demand_y\n",
    "\n",
    "dfd.drop(dfd[dfd.date == 2020.25].index, inplace = True)\n",
    "\n",
    "dfd.loc[dfd.date == 2020, 'tol_dep'] = dfd.loc[dfd.date == 2020, 'tol2']\n",
    "dfd.loc[dfd.tol_dep.isna(), 'tol_dep'] = dfd.loc[dfd.tol_dep.isna(), 'tol2']\n",
    "\n",
    "df = df.merge(dfd[['date', 'tol_dep']], how = 'left')\n",
    "\n",
    "df['tol_dep_interp'] = np.log(df.tol_dep.interpolate() / df.ngdp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure C.2: Evolution of the fiscal supply instruments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 554,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['date', 'rameynewsgdp_12_24', 'DLdep_rat_new']].to_csv('supplyiv.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.9: Demand elasticities for different types of investors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.54365561 -0.34270145]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n"
     ]
    }
   ],
   "source": [
    "# investor reg\n",
    "res_list = []\n",
    "tab = np.zeros((2, 3))\n",
    "df_temp = df.loc[(df.date < 2020)].copy()\n",
    "df_temp['ln_dom_gdp'] = np.log(np.exp(df_temp.lprivatepar) - np.exp(df_temp.ln_row_gdp))\n",
    "spread = 'spread'\n",
    "for ii, i in enumerate(['ln_row_gdp', 'ln_dom_gdp']):   \n",
    "    Fnow = 0\n",
    "    for xx in np.arange(0, 13):\n",
    "        for v in np.arange(4, 164, 4):\n",
    "            if 'rameynewsgdp_%i_%i'%(xx, v) in df_temp.columns:\n",
    "                res1 = IV2SLS(df_temp[i], add_constant(df_temp[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                               'DLdep_rat_new', 'slope', 'date', 'vix_hat']]),\n",
    "                              None, None).fit()\n",
    "                if res1.f_statistic.stat > Fnow:\n",
    "                    Fnow = res1.f_statistic.stat\n",
    "                    \n",
    "                    depvar = df_temp[spread]\n",
    "\n",
    "                    exog = add_constant(df_temp[['slope', 'date', 'vix_hat']])\n",
    "\n",
    "                    endog = df_temp[[i]].copy()\n",
    "\n",
    "                    instr = df_temp[['rameynewsgdp_%i_%i'%(xx, v)]].copy()\n",
    "                    \n",
    "                    instr['DLdep_rat_new'] = df_temp.DLdep_rat_new\n",
    "\n",
    "                    res2 = IV2SLS(depvar, exog, endog, instr).fit()\n",
    "    res_list.append(res2)\n",
    "    \n",
    "    tab[ii, 0] = res2.params[-1]\n",
    "    tab[ii, 1] = res2.params[-1] / df_temp[spread].mean()\n",
    "    \n",
    "    tab[ii, -1] = res2.nobs\n",
    "    \n",
    "tab1 = pd.DataFrame(tab, columns = ['semi_elas', 'elas', 'N'], \n",
    "                   index = ['row', 'dom'])\n",
    "\n",
    "row_dom_elas = tab1.elas.values\n",
    "print(row_dom_elas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 556,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((5, 3), ''))\n",
    "tab.columns = ['VARIABLES', 'Foreign Investors', 'Domestic Investors']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Observations', 'Demand elasticity', 'Markup']\n",
    "\n",
    "val1 = str(res_list[0].params.ln_row_gdp.round(2))\n",
    "if res_list[0].pvalues.ln_row_gdp < 0.01:\n",
    "    val1 = val1 + '***'\n",
    "elif res_list[0].pvalues.ln_row_gdp < 0.05:\n",
    "    val1 = val1 + '**'\n",
    "elif res_list[0].pvalues.ln_row_gdp < 0.1:\n",
    "    val1 = val1 + '*'\n",
    "    \n",
    "val2 = str(res_list[1].params.ln_dom_gdp.round(2))\n",
    "if res_list[1].pvalues.ln_dom_gdp < 0.01:\n",
    "    val2 = val2 + '***'\n",
    "elif res_list[1].pvalues.ln_dom_gdp < 0.05:\n",
    "    val2 = val2 + '**'\n",
    "elif res_list[1].pvalues.ln_dom_gdp < 0.1:\n",
    "    val2 = val2 + '*'\n",
    "    \n",
    "tab.iloc[0, 1:] = [val1, val2]\n",
    "tab.iloc[1, 1:] = ['(' + str(res_list[0].std_errors.ln_row_gdp.round(2)) + ')',\n",
    "                   '(' + str(res_list[1].std_errors.ln_dom_gdp.round(2)) + ')']\n",
    "tab.iloc[2, [1, 2]] = [int(res_list[0].nobs), int(res_list[1].nobs)]\n",
    "tab.iloc[3, 1:] = np.round(-1/tab1.elas.values, 2)\n",
    "tab.iloc[4, 1:] = -tab1.elas.values.round(2)\n",
    "\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\"])\n",
    "table.set_header_align(['c', 'c', 'c'])\n",
    "table.set_cols_dtype(['t', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "string = latextable.draw_latex(table, use_booktabs = True)\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb7.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()\n",
    "\n",
    "tabs_elas = []\n",
    "tabs_tstat = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 1: Baseline demand estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.8814664678432627\n",
      "mean markup: 0.6455863843478535\n",
      "[2.67571293 1.09189204]\n",
      "[0.37373217 0.91584146]\n",
      "iv R2: 0.5318670820251934\n",
      "rotator coeff and p value: -0.3384233188431744 1.1208666731588097e-07\n",
      "λ =  0 T stat = 2.6901342870765257\n",
      "0.007142327177250553\n",
      "λ =  1 T stat = -2.9077632995054428\n",
      "0.0036402375468839753\n",
      "λ =  2 T stat = -5.92015871353358\n",
      "3.2163107625968e-09\n",
      "λ =  3 T stat = -6.986364819805658\n",
      "2.820999511299974e-12\n",
      "λ =  4 T stat = -7.420731210590622\n",
      "1.1647548291745133e-13\n",
      "λ =  10 T stat = -7.931053283344931\n",
      "2.1729495524414886e-15\n",
      "λ =  20 T stat = -8.015409493669786\n",
      "1.097706735306546e-15\n",
      "λ =  30 T stat = -8.034266076118257\n",
      "9.414070167856346e-16\n"
     ]
    }
   ],
   "source": [
    "# OLS\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "depvar = dft[spread]\n",
    "\n",
    "exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "res2 = sm.OLS(depvar, exog).fit()\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "tabs_elas.append(['bl_ols',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_ols']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[[debt]])[['const']].to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_ols_bl = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "λ =  0 T stat = -4.059849178350676\n",
      "4.910442390426851e-05\n",
      "λ =  1 T stat = -6.234624023529798\n",
      "4.5286442246781185e-10\n",
      "λ =  2 T stat = -7.201762687430115\n",
      "5.943907942915448e-13\n",
      "λ =  3 T stat = -7.71632241572178\n",
      "1.1973420445323885e-14\n",
      "λ =  4 T stat = -8.028191659450682\n",
      "9.891986252042446e-16\n",
      "λ =  10 T stat = -8.693844839955329\n",
      "3.5037609653216665e-18\n",
      "λ =  20 T stat = -8.94312725697321\n",
      "3.783096936389625e-19\n",
      "λ =  30 T stat = -9.02836508107798\n",
      "1.742556298251922e-19\n"
     ]
    }
   ],
   "source": [
    "# IV\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bl = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.629303300498179\n",
      "mean markup: 0.6137592673471158\n",
      "iv R2: 0.494266132078351\n"
     ]
    }
   ],
   "source": [
    "# OLS - No interaction\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "depvar = dft[spread]\n",
    "\n",
    "exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "res2 = sm.OLS(depvar, exog).fit()\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "\n",
    "tabs_elas.append(['ni_ols',\n",
    "                  (-1/elas).mean(),\n",
    "                  (-1/elas).mean(),\n",
    "                  -elas.mean(), \n",
    "                  -elas.mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "\n",
    "res_ols_ni = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.5531872082730174\n",
      "mean markup: 0.391667323398664\n",
      "iv R2: 0.4677577711062785\n"
     ]
    }
   ],
   "source": [
    "# IV - No interaction\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "\n",
    "tabs_elas.append(['ni_iv',\n",
    "                  (-1/elas).mean(),\n",
    "                  (-1/elas).mean(),\n",
    "                  -elas.mean(), \n",
    "                  -elas.mean()])\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "\n",
    "res_iv_ni = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure 1: Evolution of prices and quantities of US public debt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure 2: Safe-asset demand in times of high and low volatility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure B.1: UK and US volatility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 561,
   "metadata": {},
   "outputs": [],
   "source": [
    "### MC graph\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "\n",
    "    for λ in [1]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "\n",
    "tab = dftt.date.to_frame('date')\n",
    "tab['z'] = z_dm\n",
    "tab['m1'] = m1_y_dm\n",
    "tab['pc'] = pc_y_dm\n",
    "tab['price'] = dftt.spread_S60_L40\n",
    "tab['markup'] = (pc_y_dm - m1_y_dm)\n",
    "tab['bin1'] = (dftt[var] >= np.quantile(dftt[var], 0.5)) + 0\n",
    "tab['bin2'] = (dftt[var] >= np.quantile(dftt[var], 0.75)) + 0\n",
    "tab['uk_vol'] = dftt.msdluk_vol_hat\n",
    "tab['debt'] = dftt.privatepar\n",
    "tab['us_vol'] = dftt.vix_hat_ma\n",
    "tab.to_csv('uk_graph2-1_resid2.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table B.5: First-stage regressions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138.621853853542 115.90586581123064\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "\n",
    "temp = add_constant(dft[['rameynewsgdp_%i_%i'%(startlag, endlag),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']])\n",
    "temp['newsXvol'] = dft['rameynewsgdp_%i_%i'%(startlag, endlag)] * dft[vb]\n",
    "temp['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "fs1 = sm.OLS(dft[debt], temp).fit()\n",
    "\n",
    "fs2 = sm.OLS(dft[debt] * dft[vb], temp).fit()\n",
    "\n",
    "sc = summary_col([fs1, fs2], stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new', 'newsXvol', 'depratXvol',\n",
    "                                    vb, 'vix_hat', 'slope', 'const'])\n",
    "\n",
    "temp1 = add_constant(dft[['slope', vb, 'vix_hat']])\n",
    "\n",
    "fsr1 = sm.OLS(dft[debt], temp1).fit()\n",
    "fsr2 = sm.OLS(dft[debt] * dft[vb], temp1).fit()\n",
    "\n",
    "FF1 = (fsr1.ssr - fs1.ssr) / (fsr1.df_resid - fs1.df_resid) / (fs1.ssr / fs1.df_resid)\n",
    "FF2 = (fsr2.ssr - fs2.ssr) / (fsr2.df_resid - fs2.df_resid) / (fs2.ssr / fs2.df_resid)\n",
    "\n",
    "print(FF1, FF2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 563,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([fs1, fs2], stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new', 'newsXvol', 'depratXvol',\n",
    "                                    vb, 'vix_hat', 'slope', 'const'])\n",
    "\n",
    "tab = sc.tables[0].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Log(debt/GDP)', 'FVD $\\\\times$ Log(debt/GDP)']\n",
    "tab.iloc[:, 0] = ['Military news', '', '$\\\\Delta$ Log Dependency', '', 'FVD $\\\\times$ Military news', '',\n",
    "                  'FVD $\\\\times$ $\\\\Delta$ Log Dependency', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'US VIX', '', 'Yield curve slope', '', 'Constant', '', 'Observations', 'R-squared']\n",
    "tab.iloc[16, [1, 2]] = [int(fs1.nobs), int(fs2.nobs)]\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\"])\n",
    "table.set_header_align(['c', 'c', 'c'])\n",
    "table.set_cols_dtype(['t', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "string = latextable.draw_latex(table, use_booktabs = True)\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabc1.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 564,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_ols_bl, res_iv_bl, res_ols_ni, res_iv_ni], stars = True, float_format = '%.2f', \n",
    "                regressor_order = ['lprivatepar', vb, 'lppXvol', 'vix_hat', 'slope', 'const'])\n",
    "\n",
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "\n",
    "tabs2 = pd.DataFrame(tabs_tstat).T\n",
    "tabs2.columns = tabs2.iloc[0]\n",
    "tabs2.drop(index = 0, inplace = True)\n",
    "tabs2 = tabs2.astype(float).round(2)\n",
    "tabs2 = tabs2.T\n",
    "\n",
    "n = int(tabs2.shape[1] / 2)\n",
    "tabs3 = np.empty((tabs2.shape[0] * 2, 1 + n), dtype = object)\n",
    "tabs2 = tabs2.reset_index()\n",
    "for i in range(tabs2.shape[0]):\n",
    "    tabs3[i*2] = tabs2.iloc[i, :n+1]\n",
    "    tabs3[i*2+1, 1:] = '(' + tabs2.iloc[i, n+1:].astype(str) + ')'\n",
    "\n",
    "tab = sc.tables[0].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'OLS', 'IV', 'OLS', 'IV']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'FVD $\\\\times$ Log(debt/gdp)', '',\n",
    "                  'US VIX', '', 'Yield curve slope', '', 'Constant', '', 'Observations', 'R-squared']\n",
    "tab.iloc[-2, 1:] = [int(res_ols_bl.nobs), int(res_iv_bl.nobs), int(res_ols_ni.nobs), int(res_iv_ni.nobs)]\n",
    "tab.iloc[-1, [2, 4]] = ['', '']\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity, high foreign vol', 'Demand elasticity, low foreign vol',\n",
    "                    'Markup, high foreign vol', 'Markup, low foreign vol']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table 2: Conduct tests for different cost elasticities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 565,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((6, 9), ''),\n",
    "                   columns = ['Cost elasticity', '$\\\\lambda = 0$', '$\\\\lambda = 1$', '$\\\\lambda = 2$',\n",
    "                              '$\\\\lambda = 3$', '$\\\\lambda = 4$', '$\\\\lambda = 10$', '$\\\\lambda = 20$',\n",
    "                              '$\\\\lambda = 30$'])\n",
    "tab.iloc[0, 0] = '\\\\emph{a. OLS} \\\\vspace{.5em}'\n",
    "tab.iloc[[1, 2], 1:] = tabs3[[0, 1], 1:]\n",
    "tab.iloc[3, 0] = '\\\\emph{b. IV} \\\\vspace{.5em}'\n",
    "tab.iloc[[4, 5], 1:] = tabs3[[2, 3], 1:]\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tab2.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()\n",
    "\n",
    "tabs_elas = []\n",
    "tabs_tstat = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Baseline IV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 566,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "from year 1926 onwards\n",
      "λ =  0 T stat = -4.059849178350676\n",
      "4.910442390426851e-05\n",
      "λ =  1 T stat = -6.234624023529798\n",
      "4.5286442246781185e-10\n",
      "λ =  2 T stat = -7.201762687430115\n",
      "5.943907942915448e-13\n",
      "λ =  3 T stat = -7.71632241572178\n",
      "1.1973420445323885e-14\n",
      "λ =  4 T stat = -8.028191659450682\n",
      "9.891986252042446e-16\n",
      "λ =  10 T stat = -8.693844839955329\n",
      "3.5037609653216665e-18\n",
      "λ =  20 T stat = -8.94312725697321\n",
      "3.783096936389625e-19\n",
      "λ =  30 T stat = -9.02836508107798\n",
      "1.742556298251922e-19\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "# print(res2.summary)\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "    print('from year %i onwards'%year)\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "                \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bl = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Short spreads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 567,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.8785408036057842\n",
      "mean markup: 0.3714813524092641\n",
      "[3.6141845  2.14722443]\n",
      "[0.27668759 0.4657175 ]\n",
      "iv R2: 0.3497120549266812\n",
      "rotator coeff and p value: -0.10018028287427537 0.40583250241853874\n",
      "λ =  0 T stat = -0.8906544937951971\n",
      "0.37311455484724043\n",
      "λ =  1 T stat = -2.86962089045664\n",
      "0.00410964202278438\n",
      "λ =  2 T stat = -4.212301500887176\n",
      "2.527818250737109e-05\n",
      "λ =  3 T stat = -5.140321362986437\n",
      "2.742689704136719e-07\n",
      "λ =  4 T stat = -5.803520474182779\n",
      "6.493681930112266e-09\n",
      "λ =  10 T stat = -7.547672416171701\n",
      "4.431057355224827e-14\n",
      "λ =  20 T stat = -8.330045000878131\n",
      "8.081163127258562e-17\n",
      "λ =  30 T stat = -8.612466867693614\n",
      "7.150496681443225e-18\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_short'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_short',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_short']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_short = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Short CD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 568,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.5383314810390076\n",
      "mean markup: 0.6671634190960682\n",
      "[1.78597056 1.29292339]\n",
      "[0.55991964 0.77344103]\n",
      "iv R2: 0.48679475040444153\n",
      "rotator coeff and p value: -0.13735898263690782 0.5014829807721564\n",
      "λ =  0 T stat = -1.494920131033704\n",
      "0.13493528774906258\n",
      "λ =  1 T stat = -3.312880574636733\n",
      "0.0009234040031429041\n",
      "λ =  2 T stat = -4.523333516403786\n",
      "6.087320425916945e-06\n",
      "λ =  3 T stat = -5.311995726304495\n",
      "1.0843114707941581e-07\n",
      "λ =  4 T stat = -5.837702589444106\n",
      "5.292552187886119e-09\n",
      "λ =  10 T stat = -7.010601009780516\n",
      "2.3729655060487404e-12\n",
      "λ =  20 T stat = -7.4142660011572294\n",
      "1.223002351600485e-13\n",
      "λ =  30 T stat = -7.538254976447132\n",
      "4.76301888696726e-14\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.spread_sa.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_sa'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[dft[spread].notna()].copy()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_shortcd',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_shortcd']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_shortcd = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Long spreads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 569,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.768885997812649\n",
      "mean markup: 0.44536042126613856\n",
      "[3.97796501 1.56691922]\n",
      "[0.25138482 0.63819499]\n",
      "iv R2: 0.34717351839664856\n",
      "rotator coeff and p value: -0.2961893311215169 0.02011301374778469\n",
      "λ =  0 T stat = -3.785202899968135\n",
      "0.00015358321962438297\n",
      "λ =  1 T stat = -7.033823958982504\n",
      "2.0094863330990894e-12\n",
      "λ =  2 T stat = -8.481857453054461\n",
      "2.216269922995504e-17\n",
      "λ =  3 T stat = -9.060889967720794\n",
      "1.2939047943677607e-19\n",
      "λ =  4 T stat = -9.300900422854516\n",
      "1.3926126807312807e-20\n",
      "λ =  10 T stat = -9.454886738402601\n",
      "3.233718656153982e-21\n",
      "λ =  20 T stat = -9.372343522870084\n",
      "7.093960518948858e-21\n",
      "λ =  30 T stat = -9.32549985841205\n",
      "1.1046167639498715e-20\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_long',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_long']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_long = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: External debt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 570,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.4153092687676327\n",
      "mean markup: 0.708725357235951\n",
      "[1.33683482 1.49329325]\n",
      "[0.74803557 0.66966083]\n",
      "iv R2: 0.21117252615764603\n",
      "rotator coeff and p value: 0.04895055484771755 0.45565590634903763\n",
      "λ =  0 T stat = 5.492955194801162\n",
      "3.952630987594053e-08\n",
      "λ =  1 T stat = 4.841960333972271\n",
      "1.285644321261055e-06\n",
      "λ =  2 T stat = 3.706717720890197\n",
      "0.00020996274122038466\n",
      "λ =  3 T stat = 3.071674094808598\n",
      "0.0021286199366269405\n",
      "λ =  4 T stat = 2.6988289101917955\n",
      "0.006958393930171916\n",
      "λ =  10 T stat = 1.9453929356038675\n",
      "0.05172770434296908\n",
      "λ =  20 T stat = 1.6755021492821744\n",
      "0.0938357497358688\n",
      "λ =  30 T stat = 1.5840028373473465\n",
      "0.11319307341798526\n"
     ]
    }
   ],
   "source": [
    "### External\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "\n",
    "dft['lprivatepar'] = dft['lexternal']\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft[debt].notna()) & (dft.DLdep_rat_new.notna())].copy()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v), 'slope',\n",
    "                                                           'DLdep_rat_new', vb, 'date', 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'date', 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'date', 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_ex',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_ex']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_ex = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Trend GDP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 571,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.6055876536934215\n",
      "mean markup: 0.4304243352986273\n",
      "[3.46660175 1.74963835]\n",
      "[0.28846694 0.57154669]\n",
      "iv R2: 0.4975259547446912\n",
      "rotator coeff and p value: -0.17671858762022835 0.033158481729527255\n",
      "λ =  0 T stat = -3.935537404770784\n",
      "8.301073350049027e-05\n",
      "λ =  1 T stat = -6.24054050967361\n",
      "4.3606144295848254e-10\n",
      "λ =  2 T stat = -7.2651824947203485\n",
      "3.7253497680206317e-13\n",
      "λ =  3 T stat = -7.809289983049452\n",
      "5.7511042411443965e-15\n",
      "λ =  4 T stat = -8.138551227946786\n",
      "4.000357825729112e-16\n",
      "λ =  10 T stat = -8.839733525263354\n",
      "9.594716591930474e-19\n",
      "λ =  20 T stat = -9.101727818316686\n",
      "8.890600841681026e-20\n",
      "λ =  30 T stat = -9.191242193964815\n",
      "3.8833066416418325e-20\n"
     ]
    }
   ],
   "source": [
    "### trend gdp\n",
    "\n",
    "dft = df.copy()\n",
    "dft.set_index('date', inplace = True)\n",
    "cycle, trend = sm.tsa.filters.hpfilter(dft.ngdp, 1600)\n",
    "\n",
    "dft['privatepar2'] = dft.privatepar_nogdp / trend\n",
    "dft['lprivatepar2'] = np.log(dft.privatepar2)\n",
    "\n",
    "dft.reset_index(inplace = True)\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = dft[dft[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "dft['lprivatepar'] = dft['lprivatepar2']\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',  'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]].copy()\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "tabs_elas.append(['iv_tr',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_tr']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_tr = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Detrended log debt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 572,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.112090028717484\n",
      "mean markup: 0.5406142040241487\n",
      "[2.85943795 1.36913827]\n",
      "[0.34971908 0.73038642]\n",
      "rotator coeff and p value: -0.23763973150398712 0.012691597976302147\n",
      "λ =  0 T stat = -1.5083358183537232\n",
      "0.13146860038308245\n",
      "λ =  1 T stat = -4.278593943071266\n",
      "1.880776093297235e-05\n",
      "λ =  2 T stat = -5.611982820166573\n",
      "2.0002130052291885e-08\n",
      "λ =  3 T stat = -6.328559355538589\n",
      "2.4746066707257574e-10\n",
      "λ =  4 T stat = -6.76023484438452\n",
      "1.3776824532079557e-11\n",
      "λ =  10 T stat = -7.66090324119983\n",
      "1.8462992413316365e-14\n",
      "λ =  20 T stat = -7.9860094646250985\n",
      "1.3937699072154235e-15\n",
      "λ =  30 T stat = -8.095056504989666\n",
      "5.723777679868219e-16\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "dft['r_privatepar_nogdp'] = dft.privatepar_nogdp / dft.pgdp\n",
    "dft['lr_privatepar_nogdp'] = np.log(dft['r_privatepar_nogdp'])\n",
    "dft['dt_lr_privatepar_nogdp'] = sm.OLS(dft['lr_privatepar_nogdp'], sm.add_constant(dft.date)).fit().resid\n",
    "dft['lprivatepar'] = dft.dt_lr_privatepar_nogdp\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_dtdebt',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_dtdebt']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_dtdebt = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 573,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_iv_bl, res_iv_short, res_iv_shortcd, res_iv_long, res_iv_ex, res_iv_tr, res_iv_dtdebt],\n",
    "                 stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['lprivatepar', vb, 'lppXvol'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Detrended debt-to-gdp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 574,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "λ =  0 T stat = -4.059849178350676\n",
      "4.910442390426851e-05\n",
      "λ =  1 T stat = -3.9916209626264534\n",
      "6.562320106643919e-05\n",
      "λ =  2 T stat = -3.9041621103713697\n",
      "9.455238966127263e-05\n",
      "λ =  3 T stat = -3.8110770675176124\n",
      "0.00013836261911854818\n",
      "λ =  4 T stat = -3.719570327746481\n",
      "0.00019956197659687602\n",
      "λ =  10 T stat = -3.2969555115198146\n",
      "0.000977389816831315\n",
      "λ =  20 T stat = -2.9446939453450436\n",
      "0.003232742662266195\n",
      "λ =  30 T stat = -2.7733936581133176\n",
      "0.005547496822610028\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "            \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for fc in [1600]:\n",
    "    cycle, trend = sm.tsa.filters.hpfilter(dft.lprivatepar, fc)\n",
    "    dft['lprivatepar_c'] = cycle\n",
    "\n",
    "    dftt = dft.copy()\n",
    "    \n",
    "    tlist1 = ['iv_f' + str(fc)]\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt['lprivatepar_c']\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt['lprivatepar_c']\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "    tabs_tstat.append(tlist1 + tlist2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table B.11: US public debt demand estimation: Alternative measures of spreads and debt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "\n",
    "tabs2 = pd.DataFrame(tabs_tstat).T\n",
    "tabs2.columns = tabs2.iloc[0]\n",
    "tabs2.drop(index = 0, inplace = True)\n",
    "tabs2 = tabs2.astype(float).round(2)\n",
    "tabs2 = tabs2.T\n",
    "\n",
    "n = int(tabs2.shape[1] / 2)\n",
    "tabs3 = np.empty((tabs2.shape[0] * 2, 1 + n), dtype = object)\n",
    "tabs2 = tabs2.reset_index()\n",
    "for i in range(tabs2.shape[0]):\n",
    "    tabs3[i*2] = tabs2.iloc[i, :n+1]\n",
    "    tabs3[i*2+1, 1:] = '(' + tabs2.iloc[i, n+1:].astype(str) + ')'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 576,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = sc.tables[0].iloc[:7, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline', 'Short Mat. (CP)', 'Short Mat. (CD)', 'Long Mat.',\n",
    "               'External Debt', 'Debt to Trend GDP', 'Detrended Log of Real Debt']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'FVD $\\\\times$ Log(debt/gdp)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_bl.nobs), int(res_iv_short.nobs), int(res_iv_shortcd.nobs),\n",
    "                   int(res_iv_long.nobs), int(res_iv_ex.nobs), int(res_iv_tr.nobs), int(res_iv_dtdebt.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity, high foreign vol', 'Demand elasticity, low foreign vol',\n",
    "                    'Markup, high foreign vol', 'Markup, low foreign vol']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', 'c', 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', 't', 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1),\n",
    "                                                           ('(5)', 1), ('(6)', 1), ('(7)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb8.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.13: Government conduct test: Alternative measures of spreads and debt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((17, 9), ''),\n",
    "                   columns = ['Cost elasticity', '$\\\\lambda = 0$', '$\\\\lambda = 1$', '$\\\\lambda = 2$',\n",
    "                              '$\\\\lambda = 3$', '$\\\\lambda = 4$', '$\\\\lambda = 10$', '$\\\\lambda = 20$',\n",
    "                              '$\\\\lambda = 30$'])\n",
    "tab.iloc[0, 0] = '\\\\emph{a. Baseline} \\\\vspace{.5em}'\n",
    "tab.iloc[[1, 2], 1:] = tabs3[[0, 1], 1:]\n",
    "tab.iloc[3, 0] = '\\\\emph{b. Different Maturity} \\\\vspace{.5em}'\n",
    "tab.iloc[4:10, 1:] = tabs3[2:8, 1:]\n",
    "tab.iloc[[4, 6, 8], 0] = ['Short (Commercial Paper)', 'Short (Certificate of Deposit)', 'Long']\n",
    "tab.iloc[10, 0] = '\\\\emph{c. Different dependent variable} \\\\vspace{.5em}'\n",
    "tab.iloc[11:17, 1:] = tabs3[10:16, 1:]\n",
    "tab.iloc[[11, 13, 15], 0] = ['Debt-to-Trend GDP', 'Detrended Log of Real Debt', 'Detrended Debt-to-GDP']\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 578,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb10.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Different controls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs_elas = []\n",
    "tabs_tstat = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "λ =  0 T stat = -4.059849178350676\n",
      "4.910442390426851e-05\n",
      "λ =  1 T stat = -6.234624023529798\n",
      "4.5286442246781185e-10\n",
      "λ =  2 T stat = -7.201762687430115\n",
      "5.943907942915448e-13\n",
      "λ =  3 T stat = -7.71632241572178\n",
      "1.1973420445323885e-14\n",
      "λ =  4 T stat = -8.028191659450682\n",
      "9.891986252042446e-16\n",
      "λ =  10 T stat = -8.693844839955329\n",
      "3.5037609653216665e-18\n",
      "λ =  20 T stat = -8.94312725697321\n",
      "3.783096936389625e-19\n",
      "λ =  30 T stat = -9.02836508107798\n",
      "1.742556298251922e-19\n"
     ]
    }
   ],
   "source": [
    "# baseline IV\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bl = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Deposits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.9048032992558097\n",
      "mean markup: 0.5596701015182264\n",
      "[2.38072237 1.43168376]\n",
      "[0.42004058 0.69847827]\n",
      "iv R2: 0.5456800062187952\n",
      "rotator coeff and p value: -0.17382068796408864 0.02838979617663399\n",
      "λ =  0 T stat = -3.2362280905524083\n",
      "0.0012112055923482585\n",
      "λ =  1 T stat = -5.843909588635372\n",
      "5.098970939058056e-09\n",
      "λ =  2 T stat = -6.9868467906237655\n",
      "2.811329512578762e-12\n",
      "λ =  3 T stat = -7.5808576014319735\n",
      "3.432782074555529e-14\n",
      "λ =  4 T stat = -7.934143505107589\n",
      "2.119529088291065e-15\n",
      "λ =  10 T stat = -8.666373623896455\n",
      "4.461013738658092e-18\n",
      "λ =  20 T stat = -8.931329845721123\n",
      "4.2091806347071107e-19\n",
      "λ =  30 T stat = -9.020577033676025\n",
      "1.8710114641988624e-19\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[(dft.date >= 1935) & (dft[var].notna()) & dft['tol_dep_interp'].notna()].copy()\n",
    "\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v), 'slope',\n",
    "                                                           'DLdep_rat_new', vb, 'vix_hat', 'tol_dep_interp']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat', 'tol_dep_interp']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat', 'tol_dep_interp']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['tol_dep',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "   \n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['tol_dep']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_tol_dep = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: BAA/AAA spread + Corp default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.707445975069771\n",
      "mean markup: 0.46057764796229256\n",
      "[3.91753864 1.5044715 ]\n",
      "[0.25526232 0.66468524]\n",
      "iv R2: 0.5947350044729125\n",
      "rotator coeff and p value: -0.2555910163774077 0.0006607785129255928\n",
      "λ =  0 T stat = -2.391364680611908\n",
      "0.01678587047823075\n",
      "λ =  1 T stat = -5.379873019383988\n",
      "7.453839555744335e-08\n",
      "λ =  2 T stat = -6.715699226918575\n",
      "1.8716650732976256e-11\n",
      "λ =  3 T stat = -7.40419208228174\n",
      "1.3195131637008646e-13\n",
      "λ =  4 T stat = -7.808942906896699\n",
      "5.7669614538170446e-15\n",
      "λ =  10 T stat = -8.6291032396806\n",
      "6.183485960024334e-18\n",
      "λ =  20 T stat = -8.916944912980012\n",
      "4.793279748782816e-19\n",
      "λ =  30 T stat = -9.012483726731928\n",
      "2.0144163392547878e-19\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "dft = dft[dft['CorpDefRate'].notna()].copy()\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat',\n",
    "                                                       'BAA_AAA_spread', 'CorpDefRate']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat', 'CorpDefRate', 'BAA_AAA_spread']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat', 'CorpDefRate',\n",
    "                                          'BAA_AAA_spread']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_bacorp',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_bacorp']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bacorp = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: No controls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 583,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.578788169833956\n",
      "mean markup: 0.49647841473111926\n",
      "[3.79066801 1.37403703]\n",
      "[0.26380575 0.72778242]\n",
      "iv R2: 0.3946006123909801\n",
      "rotator coeff and p value: -0.28964736568100186 0.001743192528857958\n",
      "λ =  0 T stat = -1.127129763676285\n",
      "0.2596876209047234\n",
      "λ =  1 T stat = -4.579987372757056\n",
      "4.650040033065434e-06\n",
      "λ =  2 T stat = -6.239537590711867\n",
      "4.3886628468017274e-10\n",
      "λ =  3 T stat = -7.098106955558569\n",
      "1.264772917312811e-12\n",
      "λ =  4 T stat = -7.596511545899274\n",
      "3.042201688257363e-14\n",
      "λ =  10 T stat = -8.571846023772999\n",
      "1.0183919579307275e-17\n",
      "λ =  20 T stat = -8.89583475389561\n",
      "5.798180200214507e-19\n",
      "λ =  30 T stat = -9.000316165614665\n",
      "2.250685829550203e-19\n"
     ]
    }
   ],
   "source": [
    "### without slope and vix\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[vb, debt]])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_lc',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_lc']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_lc = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 584,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_iv_bl, res_tol_dep, res_iv_bacorp, res_iv_lc],\n",
    "                 stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['lprivatepar', vb, 'lppXvol'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.14: US public debt demand estimation: Different controls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 585,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "\n",
    "tabs2 = pd.DataFrame(tabs_tstat).T\n",
    "tabs2.columns = tabs2.iloc[0]\n",
    "tabs2.drop(index = 0, inplace = True)\n",
    "tabs2 = tabs2.astype(float).round(2)\n",
    "tabs2 = tabs2.T\n",
    "\n",
    "n = int(tabs2.shape[1] / 2)\n",
    "tabs3 = np.empty((tabs2.shape[0] * 2, 1 + n), dtype = object)\n",
    "tabs2 = tabs2.reset_index()\n",
    "for i in range(tabs2.shape[0]):\n",
    "    tabs3[i*2] = tabs2.iloc[i, :n+1]\n",
    "    tabs3[i*2+1, 1:] = '(' + tabs2.iloc[i, n+1:].astype(str) + ')'\n",
    "\n",
    "tab = sc.tables[0].iloc[:7, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline', 'Control for Deposits',\n",
    "               'Control for BAA/AAA spread + Corp default',\n",
    "               'No controls']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'FVD $\\\\times$ Log(debt/gdp)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_bl.nobs), int(res_tol_dep.nobs),\n",
    "                    int(res_iv_bacorp.nobs), int(res_iv_lc.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity, high foreign vol', 'Demand elasticity, low foreign vol',\n",
    "                    'Markup, high foreign vol', 'Markup, low foreign vol']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb11.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.16: Government conduct test: Different controls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((10, 9), ''),\n",
    "                   columns = ['Cost elasticity', '$\\\\lambda = 0$', '$\\\\lambda = 1$', '$\\\\lambda = 2$',\n",
    "                              '$\\\\lambda = 3$', '$\\\\lambda = 4$', '$\\\\lambda = 10$', '$\\\\lambda = 20$',\n",
    "                              '$\\\\lambda = 30$'])\n",
    "tab.iloc[0, 0] = '\\\\emph{a. Baseline} \\\\vspace{.5em}'\n",
    "tab.iloc[[1, 2], 1:] = tabs3[[0, 1], 1:]\n",
    "tab.iloc[3, 0] = '\\\\emph{b. Different controls} \\\\vspace{.5em}'\n",
    "tab.iloc[4:10, 1:] = tabs3[2:8, 1:]\n",
    "tab.iloc[[4, 6, 8], 0] = ['Control for Deposits',\n",
    "                          'Control for BAA/AAA spread + Corp. Default', 'No Controls']\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb13.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Different time samples / instruments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 587,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "λ =  0 T stat = -4.059849178350676\n",
      "4.910442390426851e-05\n",
      "λ =  1 T stat = -6.234624023529798\n",
      "4.5286442246781185e-10\n",
      "λ =  2 T stat = -7.201762687430115\n",
      "5.943907942915448e-13\n",
      "λ =  3 T stat = -7.71632241572178\n",
      "1.1973420445323885e-14\n",
      "λ =  4 T stat = -8.028191659450682\n",
      "9.891986252042446e-16\n",
      "λ =  10 T stat = -8.693844839955329\n",
      "3.5037609653216665e-18\n",
      "λ =  20 T stat = -8.94312725697321\n",
      "3.783096936389625e-19\n",
      "λ =  30 T stat = -9.02836508107798\n",
      "1.742556298251922e-19\n"
     ]
    }
   ],
   "source": [
    "tabs_elas = []\n",
    "tabs_tstat = []\n",
    "\n",
    "# baseline IV\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "            \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bl = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: ZLB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 588,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.4948821178790173\n",
      "mean markup: 0.4424539031156395\n",
      "[3.26019101 1.72957323]\n",
      "[0.30673049 0.57817731]\n",
      "iv R2: 0.5021151085543083\n",
      "rotator coeff and p value: -0.17169098346601053 0.0893962358727629\n",
      "λ =  0 T stat = -4.000388845932504\n",
      "6.323848589018257e-05\n",
      "λ =  1 T stat = -6.193529068231488\n",
      "5.883183261720498e-10\n",
      "λ =  2 T stat = -7.151079303962085\n",
      "8.609822871338186e-13\n",
      "λ =  3 T stat = -7.65677049409783\n",
      "1.9066692382734874e-14\n",
      "λ =  4 T stat = -7.962343455375591\n",
      "1.6881111724324362e-15\n",
      "λ =  10 T stat = -8.613750717916865\n",
      "7.07083510565029e-18\n",
      "λ =  20 T stat = -8.85820282169872\n",
      "8.131467390953259e-19\n",
      "λ =  30 T stat = -8.941984326992095\n",
      "3.822438155050797e-19\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna()) & (dft.ZLB == 0)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_zlb',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_zlb']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_zlb = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Post 1945"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.8622216737501525\n",
      "mean markup: 0.40073683003356486\n",
      "[3.8915399  1.83976557]\n",
      "[0.25696769 0.54354751]\n",
      "iv R2: 0.48864570714072675\n",
      "rotator coeff and p value: -0.18334400748871216 0.0515442808320532\n",
      "λ =  0 T stat = -4.021805668520826\n",
      "5.7753698180889256e-05\n",
      "λ =  1 T stat = -6.145804608214537\n",
      "7.955917275369525e-10\n",
      "λ =  2 T stat = -7.088049000005916\n",
      "1.3601581646944939e-12\n",
      "λ =  3 T stat = -7.5902770313754475\n",
      "3.1922224974073753e-14\n",
      "λ =  4 T stat = -7.895425610442467\n",
      "2.8932620043723898e-15\n",
      "λ =  10 T stat = -8.550203982099728\n",
      "1.2287116800754458e-17\n",
      "λ =  20 T stat = -8.797357302110296\n",
      "1.4007575468838512e-18\n",
      "λ =  30 T stat = -8.882225726158248\n",
      "6.553568137739685e-19\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1945)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_p45',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_p45']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_p45 = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Pre 2008"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 590,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.1389700068478126\n",
      "mean markup: 0.47725327623774066\n",
      "[2.44571873 1.8343223 ]\n",
      "[0.40887776 0.54516047]\n",
      "iv R2: 0.5094062905535617\n",
      "rotator coeff and p value: -0.08671982357262134 0.44878616805971727\n",
      "λ =  0 T stat = -4.882827488807201\n",
      "1.0457535586119816e-06\n",
      "λ =  1 T stat = -6.659753873152689\n",
      "2.7428652814485325e-11\n",
      "λ =  2 T stat = -7.408702547761565\n",
      "1.2754100691301687e-13\n",
      "λ =  3 T stat = -7.802643931629802\n",
      "6.0623423301509124e-15\n",
      "λ =  4 T stat = -8.041392260574533\n",
      "8.882342316336156e-16\n",
      "λ =  10 T stat = -8.55596293367897\n",
      "1.1688885690060382e-17\n",
      "λ =  20 T stat = -8.75269037915532\n",
      "2.0832566979446534e-18\n",
      "λ =  30 T stat = -8.820809193050952\n",
      "1.1363551055143479e-18\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.date < 2008) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_p08',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_p08']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_p08 = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Post 1972"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 591,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.7945037478101702\n",
      "mean markup: 0.6243621012298018\n",
      "[1.20619749 2.38281   ]\n",
      "[0.82905163 0.41967257]\n",
      "iv R2: 0.5191693278563421\n",
      "rotator coeff and p value: -0.2827871272764012 0.010398059095186072\n",
      "λ =  0 T stat = 1.7853693458308848\n",
      "0.07420142745795491\n",
      "λ =  1 T stat = 1.0935774273782268\n",
      "0.27414035363560674\n",
      "λ =  2 T stat = 0.45890157851140373\n",
      "0.6463048451765205\n",
      "λ =  3 T stat = -0.07531192100958636\n",
      "0.9399665368264583\n",
      "λ =  4 T stat = -0.5053225249168046\n",
      "0.6133323246199684\n",
      "λ =  10 T stat = -1.7644441811604747\n",
      "0.07765722529457929\n",
      "λ =  20 T stat = -2.3263346375891176\n",
      "0.020000705570450285\n",
      "λ =  30 T stat = -2.5174660443804147\n",
      "0.011820236240940491\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1972)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_p72',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_p72']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_p72 = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Military news shock only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.7944966381846523\n",
      "mean markup: 0.565538280061232\n",
      "[2.01233547 1.57793921]\n",
      "[0.49693504 0.63373798]\n",
      "iv R2: 0.5062516844698874\n",
      "rotator coeff and p value: -0.08540216221024263 0.5545410467042078\n",
      "λ =  0 T stat = -4.927510907290684\n",
      "8.328377570769578e-07\n",
      "λ =  1 T stat = -7.099593102736081\n",
      "1.2512473005582474e-12\n",
      "λ =  2 T stat = -7.945467255972833\n",
      "1.9346015405541583e-15\n",
      "λ =  3 T stat = -8.368897560638265\n",
      "5.815978235219412e-17\n",
      "λ =  4 T stat = -8.61756074532169\n",
      "6.8395481544711704e-18\n",
      "λ =  10 T stat = -9.131450558748401\n",
      "6.758753221016387e-20\n",
      "λ =  20 T stat = -9.31942002385861\n",
      "1.1697796858645937e-20\n",
      "λ =  30 T stat = -9.383347712347394\n",
      "6.391059500111881e-21\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                            'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                \n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_ramey',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_ramey']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag)]]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_ramey = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Dependency ratio only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 593,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 3.191154931332759\n",
      "mean markup: 0.3760656668793093\n",
      "[4.49959033 1.89041621]\n",
      "[0.22224245 0.52898404]\n",
      "iv R2: 0.4870001185083601\n",
      "rotator coeff and p value: -0.19148999440099032 0.031088043664964992\n",
      "λ =  0 T stat = -4.317754286474729\n",
      "1.5762476294333627e-05\n",
      "λ =  1 T stat = -6.48471081624591\n",
      "8.890213636193758e-11\n",
      "λ =  2 T stat = -7.305368099066945\n",
      "2.7650871101882493e-13\n",
      "λ =  3 T stat = -7.7038313281442665\n",
      "1.320461194920301e-14\n",
      "λ =  4 T stat = -7.932361574826761\n",
      "2.1501732606552135e-15\n",
      "λ =  10 T stat = -8.38732689184128\n",
      "4.973207078426181e-17\n",
      "λ =  20 T stat = -8.546135071345505\n",
      "1.2727904893576719e-17\n",
      "λ =  30 T stat = -8.598965669339334\n",
      "8.04377423365874e-18\n"
     ]
    }
   ],
   "source": [
    "### Deprat only\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "depvar = dft[[spread]]\n",
    "\n",
    "exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "instr = add_constant(dft[[ 'slope', vb, 'DLdep_rat_new', 'vix_hat']])\n",
    "instr['depXvol'] = dft['DLdep_rat_new'] * dft[vb]\n",
    "\n",
    "res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_dep',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_dep']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_dep = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Robustness specifications: Blanchard-Perotti only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 594,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.3023365831605949\n",
      "mean markup: 0.9745462296852541\n",
      "[1.90470017 0.70351631]\n",
      "[0.52501702 1.42143115]\n",
      "iv R2: 0.4596534129583111\n",
      "rotator coeff and p value: -0.5596057028151272 0.0015282889537762496\n",
      "λ =  0 T stat = nan\n",
      "nan\n",
      "λ =  1 T stat = nan\n",
      "nan\n",
      "λ =  2 T stat = nan\n",
      "nan\n",
      "λ =  3 T stat = nan\n",
      "nan\n",
      "λ =  4 T stat = nan\n",
      "nan\n",
      "λ =  10 T stat = nan\n",
      "nan\n",
      "λ =  20 T stat = nan\n",
      "nan\n",
      "λ =  30 T stat = nan\n",
      "nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/907475843.py:76: RuntimeWarning: invalid value encountered in log\n",
      "  m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'bpshock_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['bpshock_%i_%i'%(xx, v),\n",
    "                                                            'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'bpshock_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['bpshock_%i_%i'%(xx, v)] * dft[vb]\n",
    "                \n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_bp',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_bp']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['bpshock_%i_%i'%(startlag, endlag)]]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bp = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_iv_bl, res_iv_zlb,\n",
    "                  res_iv_p45, res_iv_p08, res_iv_p72,\n",
    "                  res_iv_ramey, res_iv_dep, res_iv_bp], stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['lprivatepar', vb, 'lppXvol'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing instrument: UK volatility indicator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "λ =  0 T stat = 0.17870112541666014\n",
      "0.8581723854553145\n",
      "λ =  1 T stat = -2.1487320949309807\n",
      "0.031655642972941204\n",
      "λ =  2 T stat = -3.1799174648455946\n",
      "0.0014731700465793927\n",
      "λ =  3 T stat = -3.6972483139456287\n",
      "0.0002179491288276255\n",
      "λ =  4 T stat = -3.9962404502827535\n",
      "6.435636845502026e-05\n",
      "λ =  10 T stat = -4.592923243899663\n",
      "4.3707985581512096e-06\n",
      "λ =  20 T stat = -4.801578000864713\n",
      "1.5742016701080196e-06\n",
      "λ =  30 T stat = -4.8711733776571355\n",
      "1.109374363495517e-06\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "# dft[vb] =  np.log(dft[var])\n",
    "# vb = var\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',  'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "# print(res2.summary)\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bin_test']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "#         vb = 'vix_hat_bin'\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[vb].to_numpy()\n",
    "#         z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        # print(np.corrcoef(m1_y_dm, z_dm)[0, 1], np.corrcoef(pc_y_dm, z_dm)[0, 1])\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "# res_iv_bl = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 597,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 598,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs2 = pd.DataFrame(tabs_tstat).T\n",
    "tabs2.columns = tabs2.iloc[0]\n",
    "tabs2.drop(index = 0, inplace = True)\n",
    "tabs2 = tabs2.astype(float).round(2)\n",
    "tabs2 = tabs2.T\n",
    "\n",
    "n = int(tabs2.shape[1] / 2)\n",
    "tabs3 = np.empty((tabs2.shape[0] * 2, 1 + n), dtype = object)\n",
    "tabs2 = tabs2.reset_index()\n",
    "for i in range(tabs2.shape[0]):\n",
    "    tabs3[i*2] = tabs2.iloc[i, :n+1]\n",
    "    tabs3[i*2+1, 1:] = '(' + tabs2.iloc[i, n+1:].astype(str) + ')'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.17: US public debt demand estimation: Additional robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 599,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = sc.tables[0].iloc[:7, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline',\n",
    "               'No ZLB', 'Post War', 'Pre GFC', 'High External Debt', 'Military', 'Dependency', 'BP shock']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'FVD $\\\\times$ Log(debt/gdp)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_bl.nobs),\n",
    "                    int(res_iv_zlb.nobs), int(res_iv_p45.nobs), int(res_iv_p08.nobs), int(res_iv_p72.nobs),\n",
    "                  int(res_iv_ramey.nobs), int(res_iv_dep.nobs), int(res_iv_bp.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity, high foreign vol', 'Demand elasticity, low foreign vol',\n",
    "                    'Markup, high foreign vol', 'Markup, low foreign vol']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 600,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', 'c', 'c', \"c\", \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', 't', 't', \"t\", 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', \"t\", 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1),\n",
    "                                                           ('(5)', 1), ('(6)', 1), ('(7)', 1), ('(8)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb11_b.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table B.19: Government conduct test: Additional robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 601,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((20, 9), ''),\n",
    "                   columns = ['Cost elasticity', '$\\\\lambda = 0$', '$\\\\lambda = 1$', '$\\\\lambda = 2$',\n",
    "                              '$\\\\lambda = 3$', '$\\\\lambda = 4$', '$\\\\lambda = 10$', '$\\\\lambda = 20$',\n",
    "                              '$\\\\lambda = 30$'])\n",
    "tab.iloc[0, 0] = '\\\\emph{a. Baseline} \\\\vspace{.5em}'\n",
    "tab.iloc[[1, 2], 1:] = tabs3[[0, 1], 1:]\n",
    "# tab.iloc[3, 0] = '\\\\emph{b. Different controls} \\\\vspace{.5em}'\n",
    "# tab.iloc[4:10, 1:] = tabs3[2:8, 1:]\n",
    "# tab.iloc[[4, 6, 8], 0] = ['Control for Deposits + UK Debt', 'Control for BAA/AAA spread + Corp. Default', 'No Controls']\n",
    "tab.iloc[3, 0] = '\\\\emph{c. Other time samples} \\\\vspace{.5em}'\n",
    "tab.iloc[4:12, 1:] = tabs3[2:10, 1:]\n",
    "tab.iloc[[4, 6, 8, 10], 0] = ['No ZLB', 'Post War', 'Pre GFC', 'High External Debt']\n",
    "tab.iloc[12, 0] = '\\\\emph{d. Different supply instruments} \\\\vspace{.5em}'\n",
    "tab.iloc[13:17, 1:] = tabs3[10:14, 1:]\n",
    "tab.iloc[[13, 15], 0] = ['Military news', '$\\\\Delta$ dependency ratio']\n",
    "tab.iloc[17, 0] = '\\\\emph{e. Different testing instrument} \\\\vspace{.5em}'\n",
    "tab.iloc[18:, 1:] = tabs3[16:, 1:]\n",
    "tab.iloc[18, 0] = 'UK vol indicator'\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 602,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb13_b.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs_elas = []\n",
    "tabs_tstat = []\n",
    "tabs_fstat = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alternative rotators specifications: Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 604,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n",
      "λ =  0 T stat = -4.059849178350676\n",
      "4.910442390426851e-05\n",
      "λ =  1 T stat = -6.234624023529798\n",
      "4.5286442246781185e-10\n",
      "λ =  2 T stat = -7.201762687430115\n",
      "5.943907942915448e-13\n",
      "λ =  3 T stat = -7.71632241572178\n",
      "1.1973420445323885e-14\n",
      "λ =  4 T stat = -8.028191659450682\n",
      "9.891986252042446e-16\n",
      "λ =  10 T stat = -8.693844839955329\n",
      "3.5037609653216665e-18\n",
      "λ =  20 T stat = -8.94312725697321\n",
      "3.783096936389625e-19\n",
      "λ =  30 T stat = -9.02836508107798\n",
      "1.742556298251922e-19\n"
     ]
    }
   ],
   "source": [
    "### Baseline\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv']\n",
    "    tlist2 = []\n",
    "    \n",
    "    flist1 = ['bl_iv']\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "        \n",
    "        π1 = m1_y_dm.dot(z_dm) /  z_dm.dot(z_dm)\n",
    "        π2 = pc_y_dm.dot(z_dm) /  z_dm.dot(z_dm)\n",
    "        ϕ1 = W * z_dm * (m1_y_dm - z_dm.dot(π1))\n",
    "        ϕ2 = W * z_dm * (pc_y_dm - z_dm.dot(π1))\n",
    "\n",
    "        V_AR11 = (ϕ1 * ϕ1).mean()\n",
    "        V_AR22 = (ϕ2 * ϕ2).mean()\n",
    "        V_AR12 = (ϕ1 * ϕ2).mean()\n",
    "\n",
    "        σ1_2 = V_AR11 / W\n",
    "        σ2_2 = V_AR22 / W\n",
    "        σ12 = V_AR12 / W\n",
    "        ρ_2 = (σ1_2 - σ2_2)**2 / ((σ1_2 + σ2_2)**2 - 4 * σ12**2)\n",
    "\n",
    "        F = (1-ρ_2) * len(z_dm) / 2 * (σ2_2 * g_m1**2 * W + σ1_2 * g_pc**2 * W\n",
    "                                       - 2 * σ12 * g_m1 * g_pc * W) / (σ1_2 * σ2_2 - σ12**2)\n",
    "        \n",
    "        flist1.append(F)\n",
    "tabs_fstat.append(flist1)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bl = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications: Investor share"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 605,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "λ =  0 T stat = -12.450095725505184\n",
      "1.396775377478473e-35\n",
      "λ =  1 T stat = -8.998690519891014\n",
      "2.2842583913233275e-19\n",
      "λ =  2 T stat = -5.744789561246188\n",
      "9.203537378434413e-09\n",
      "λ =  3 T stat = -4.015579664422506\n",
      "5.9299863304906935e-05\n",
      "λ =  4 T stat = -3.022592929497161\n",
      "0.0025061911968786567\n",
      "λ =  10 T stat = -1.067093486907527\n",
      "0.2859296226271578\n",
      "λ =  20 T stat = -0.38429764860070004\n",
      "0.7007578505920029\n",
      "λ =  30 T stat = -0.15505963575885356\n",
      "0.8767743032363854\n"
     ]
    }
   ],
   "source": [
    "elas_qest = row_dom_elas\n",
    "\n",
    "### elas in parts\n",
    "df['row_shareq_ma'] = df.row_share.rolling(4).mean()\n",
    "df['elas_t'] = df.row_shareq_ma * elas_qest[0] + (1 - df.row_shareq_ma) * elas_qest[1]\n",
    "spread = 'spread'\n",
    "debt = 'lprivatepar'\n",
    "var = 'row_shareq_ma'\n",
    "elas = 'elas_t'\n",
    "\n",
    "dft = df[(df.date >= 1940)][[elas, spread, debt, var, 'date']].copy().dropna()\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['investor_share']\n",
    "    tlist2 = []\n",
    "    \n",
    "    flist1 = ['investor_share']\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt[elas] * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[spread])[['const']].to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "        \n",
    "        π1 = m1_y_dm.dot(z_dm) /  z_dm.dot(z_dm)\n",
    "        π2 = pc_y_dm.dot(z_dm) /  z_dm.dot(z_dm)\n",
    "        ϕ1 = W * z_dm * (m1_y_dm - z_dm.dot(π1))\n",
    "        ϕ2 = W * z_dm * (pc_y_dm - z_dm.dot(π1))\n",
    "\n",
    "        V_AR11 = (ϕ1 * ϕ1).mean()\n",
    "        V_AR22 = (ϕ2 * ϕ2).mean()\n",
    "        V_AR12 = (ϕ1 * ϕ2).mean()\n",
    "\n",
    "        σ1_2 = V_AR11 / W\n",
    "        σ2_2 = V_AR22 / W\n",
    "        σ12 = V_AR12 / W\n",
    "        ρ_2 = (σ1_2 - σ2_2)**2 / ((σ1_2 + σ2_2)**2 - 4 * σ12**2)\n",
    "\n",
    "        F = (1-ρ_2) * len(z_dm) / 2 * (σ2_2 * g_m1**2 * W + σ1_2 * g_pc**2 * W\n",
    "                                       - 2 * σ12 * g_m1 * g_pc * W) / (σ1_2 * σ2_2 - σ12**2)\n",
    "        \n",
    "        flist1.append(F)\n",
    "tabs_fstat.append(flist1)\n",
    "tabs_tstat.append(tlist1 + tlist2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[(df.date >= 1940)][['date', 'elas_t', 'row_shareq_ma']].dropna().to_csv('row_share2.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications:  Residualize with hpcycle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.9809743885406648\n",
      "mean markup: 0.36478413696142375\n",
      "[3.82615028 2.1357985 ]\n",
      "[0.26135931 0.46820896]\n",
      "iv R2: 0.41110348747111436\n",
      "rotator coeff and p value: -0.11855826580083828 0.07165018814597268\n",
      "λ =  0 T stat = -5.6348278789517225\n",
      "1.7523326522315008e-08\n",
      "λ =  1 T stat = -8.013113330404314\n",
      "1.1184049101942263e-15\n",
      "λ =  2 T stat = -8.682943312838892\n",
      "3.856566217523051e-18\n",
      "λ =  3 T stat = -8.93885488824127\n",
      "3.932238087952553e-19\n",
      "λ =  4 T stat = -9.060656320372694\n",
      "1.296679593209173e-19\n",
      "λ =  10 T stat = -9.235934040324658\n",
      "2.5604222283667385e-20\n",
      "λ =  20 T stat = -9.270300588482302\n",
      "1.8562206394908426e-20\n",
      "λ =  30 T stat = -9.277739628322022\n",
      "1.7311193392816197e-20\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma']])\n",
    "XX['rgdp_cyc'] = sm.tsa.filters.hpfilter(dft.rgdp, 1600)[0]\n",
    "XX = XX.to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[(dft[spread] < dft[spread].quantile(0.95))].copy()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['iv_resid2',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['iv_resid2']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_resid2 = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications: Non Residualized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.9566897112626447\n",
      "mean markup: 0.40113487276643517\n",
      "[4.13250532 1.78779066]\n",
      "[0.24198396 0.5593496 ]\n",
      "iv R2: 0.5078813744268227\n",
      "rotator coeff and p value: -0.19812229316988336 0.017021351815119667\n",
      "λ =  0 T stat = -4.125075966994587\n",
      "3.7061240777866805e-05\n",
      "λ =  1 T stat = -6.2532685556650796\n",
      "4.019500675253583e-10\n",
      "λ =  2 T stat = -7.21619374197473\n",
      "5.346287351910368e-13\n",
      "λ =  3 T stat = -7.7323146776243465\n",
      "1.0560849652057158e-14\n",
      "λ =  4 T stat = -8.046108972498738\n",
      "8.546789582262501e-16\n",
      "λ =  10 T stat = -8.716818483293368\n",
      "2.861283176338074e-18\n",
      "λ =  20 T stat = -8.967485249777281\n",
      "3.033611074793326e-19\n",
      "λ =  30 T stat = -9.052987035293844\n",
      "1.3910981782306255e-19\n"
     ]
    }
   ],
   "source": [
    "### Non Residualized\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df.copy()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['resid',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['resid']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_resid = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications: 66 perc. cutoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 609,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.588525877460455\n",
      "mean markup: 0.4667984189325761\n",
      "[3.25767096 1.28514763]\n",
      "[0.30696777 0.77812072]\n",
      "iv R2: 0.5157994018887158\n",
      "rotator coeff and p value: -0.29412730661139935 0.0029367281906364936\n",
      "λ =  0 T stat = 0.27078202488785763\n",
      "0.7865586856278973\n",
      "λ =  1 T stat = -3.5106521811635165\n",
      "0.0004470088627808841\n",
      "λ =  2 T stat = -5.411622567729961\n",
      "6.245622242142104e-08\n",
      "λ =  3 T stat = -6.442366134927929\n",
      "1.1762509899855472e-10\n",
      "λ =  4 T stat = -7.064213003213979\n",
      "1.615288126844224e-12\n",
      "λ =  10 T stat = -8.361464325079993\n",
      "6.194452906959363e-17\n",
      "λ =  20 T stat = -8.830202642611079\n",
      "1.0448624931764748e-18\n",
      "λ =  30 T stat = -8.98774236842742\n",
      "2.523607676818605e-19\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.66)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',  'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "            \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bin_66',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bin_66']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_bin_66 = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications: 75 perc. cutoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.5485965140860523\n",
      "mean markup: 0.4455428428370117\n",
      "[2.96667845 1.29926932]\n",
      "[0.33707731 0.76966337]\n",
      "iv R2: 0.5198341969022123\n",
      "rotator coeff and p value: -0.27005109877376643 0.008636917213657343\n",
      "λ =  0 T stat = -0.5512328384209908\n",
      "0.581474071685001\n",
      "λ =  1 T stat = -4.123763483570926\n",
      "3.727316365156471e-05\n",
      "λ =  2 T stat = -5.677803302035012\n",
      "1.3643547088726907e-08\n",
      "λ =  3 T stat = -6.468506104985895\n",
      "9.89765196172512e-11\n",
      "λ =  4 T stat = -6.932392900454308\n",
      "4.137807973231271e-12\n",
      "λ =  10 T stat = -7.881340663661787\n",
      "3.238869406118162e-15\n",
      "λ =  20 T stat = -8.223115142231634\n",
      "1.9828430347772405e-16\n",
      "λ =  30 T stat = -8.338571290779322\n",
      "7.519349160035254e-17\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.75)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',  'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "            \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bin_75',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bin_75']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_bin_75 = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications: 8 quarters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 611,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/424947098.py:13: FutureWarning: Series.dt.weekofyear and Series.dt.week have been deprecated. Please use Series.dt.isocalendar().week instead.\n",
      "  df21['w'] = df21.date.dt.week\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/424947098.py:20: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df21 = df21a[df21.groupby(['y', 'w']).size() > 1].reset_index()\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/424947098.py:79: FutureWarning: In a future version of pandas all arguments of DataFrame.drop_duplicates except for the argument 'subset' will be keyword-only.\n",
      "  df21.drop_duplicates('date' ,'last', inplace = True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7377482163954748\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/424947098.py:91: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "  df2['vix_hat'] = np.nan\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n"
     ]
    }
   ],
   "source": [
    "df2 = pd.read_stata('quarterly.dta')\n",
    "\n",
    "for series in ['msdluk']:\n",
    "    \n",
    "    df21 = pd.read_excel('ftse.xlsx', sheet_name = series)\n",
    "    df21.columns = ['date', series, 'n0', 'n1']\n",
    "    df21 = df21[['date', series]].copy()\n",
    "\n",
    "    df21['date'] = pd.to_datetime(df21.date)\n",
    "    df21['q'] = df21.date.dt.quarter\n",
    "    df21['y'] = df21.date.dt.year\n",
    "\n",
    "    df21['w'] = df21.date.dt.week\n",
    "\n",
    "    df21 = df21[df21[series].notna()].copy()\n",
    "\n",
    "    df21 = df21[df21.date >= '1972'].copy()\n",
    "\n",
    "    df21a = df21.set_index(['y', 'w'])\n",
    "    df21 = df21a[df21.groupby(['y', 'w']).size() > 1].reset_index()\n",
    "\n",
    "    df21a = df21.sort_values('date').groupby(['y', 'w'])[series].agg(['first', 'last']).reset_index()\n",
    "\n",
    "    df21a['ret'] = (df21a['last'] - df21a['first']) / df21a['first']\n",
    "\n",
    "    df21a['formatted_date'] = df21a.y * 1000 + df21a.w * 10 + 0\n",
    "    df21a['date'] = pd.to_datetime(df21a['formatted_date'], format='%Y%W%w')\n",
    "\n",
    "    df21a['q'] = df21a.date.dt.quarter\n",
    "\n",
    "    df21 = df21a.drop(df21a.iloc[[0, -1]].index)\n",
    "\n",
    "    tab = []\n",
    "    for year in range(df21.y.min() + 2, df21.y.max() + 1):\n",
    "        for quarter in range(1, 5):\n",
    "            if quarter == 1:\n",
    "                ind = ((df21.y == year) & (df21.q == 1)) | ((df21.y == year-1) & (df21.q == 4)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 3)) | ((df21.y == year-1) & (df21.q == 2)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 1)) | ((df21.y == year-2) & (df21.q == 4)) \\\n",
    "                    | ((df21.y == year-2) & (df21.q == 3)) | ((df21.y == year-2) & (df21.q == 2))\n",
    "            elif quarter == 2:\n",
    "                ind = ((df21.y == year) & (df21.q == 2)) | ((df21.y == year) & (df21.q == 1)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 4)) | ((df21.y == year-1) & (df21.q == 3)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 2)) | ((df21.y == year-1) & (df21.q == 1)) \\\n",
    "                    | ((df21.y == year-2) & (df21.q == 4)) | ((df21.y == year-2) & (df21.q == 3))\n",
    "            elif quarter == 3:\n",
    "                ind = ((df21.y == year) & (df21.q == 3)) | ((df21.y == year) & (df21.q == 2)) \\\n",
    "                    | ((df21.y == year) & (df21.q == 1)) | ((df21.y == year-1) & (df21.q == 4)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 3)) | ((df21.y == year-1) & (df21.q == 2)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 1)) | ((df21.y == year-2) & (df21.q == 4))\n",
    "            elif quarter == 4:\n",
    "                ind = ((df21.y == year) & (df21.q == 4)) | ((df21.y == year) & (df21.q == 3)) \\\n",
    "                    | ((df21.y == year) & (df21.q == 2)) | ((df21.y == year) & (df21.q == 1)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 4)) | ((df21.y == year-1) & (df21.q == 3)) \\\n",
    "                    | ((df21.y == year-1) & (df21.q == 2)) | ((df21.y == year-1) & (df21.q == 1))\n",
    "            tab.append([year, quarter, df21.loc[ind].ret.std()])\n",
    "\n",
    "    df21 = pd.DataFrame(tab, columns = ['y', 'q', series + '_vol'])\n",
    "\n",
    "    df21['date'] = df21.y + (df21.q - 1) * 0.25\n",
    "    \n",
    "    df2 = df2.merge(df21[['date', series + '_vol']], on = 'date', how = 'left')\n",
    "    \n",
    "df21 = pd.read_csv('MSPIUKM.csv')\n",
    "df21.columns = ['date', 'uks']\n",
    "\n",
    "df21['uks_L'] = df21.uks.shift()\n",
    "df21['ret'] = df21.uks / df21.uks_L - 1\n",
    "\n",
    "df21.date = pd.to_datetime(df21.date)\n",
    "\n",
    "df21['q'] = df21.date.dt.quarter\n",
    "df21['y'] = df21.date.dt.year\n",
    "\n",
    "df21['ukm_vol'] = df21.ret.rolling(24).std()\n",
    "\n",
    "df21['date'] = df21.y + (df21.q - 1) * 0.25\n",
    "\n",
    "df21.drop_duplicates('date' ,'last', inplace = True)\n",
    "\n",
    "df2 = df2.merge(df21[['date', 'ukm' + '_vol']], on = 'date', how = 'left')\n",
    "    \n",
    "df21 = pd.read_csv('age_dep.csv', index_col = 0)\n",
    "df2 = df2.merge(df21, on = 'date', how = 'left')\n",
    "df2.DLdep_rat_new = np.log(df2.age_dep / df2.age_dep.shift())\n",
    "\n",
    "df21 = pd.read_excel('TVR.xlsx', sheet_name = 'Data_plots')\n",
    "\n",
    "df2 = df2.merge(df21[['date', 'VIX', 'EPU']], on = 'date', how = 'left')\n",
    "\n",
    "df2['vix_hat'] = np.nan\n",
    "\n",
    "df2t = df2[df2.date >= 1926].copy()\n",
    "exog = df2t[['vol']].copy()\n",
    "\n",
    "res = IV2SLS(df2t.VIX, add_constant(exog), None, None).fit()\n",
    "\n",
    "df2.loc[df2[df2.date >= 1926].index, 'vix_hat'] = add_constant(exog) @ res.params\n",
    "df2.loc[df2[df2.VIX.notna()].index, 'vix_hat'] = df2.loc[df2[df2.VIX.notna()].index, 'VIX']\n",
    "\n",
    "for series in ['msdluk_vol']:\n",
    "    df2t = df2[df2.date >= 1926].copy()\n",
    "    exog = (df2t[['ukm_vol']]).copy()\n",
    "\n",
    "    res = IV2SLS((df2t[series]), add_constant(exog), None, None).fit()\n",
    "    \n",
    "    print(res.rsquared)\n",
    "\n",
    "    df2[series + '_hat'] = np.nan\n",
    "    \n",
    "    df2.loc[(add_constant(exog) @ res.params).index, series + '_hat'] = (add_constant(exog) @ res.params)\n",
    "    df2.loc[df2[df2[series].notna()].index, series + '_hat'] = df2.loc[df2[df2[series].notna()].index, series]\n",
    "    \n",
    "df2 = df2[df2.spread_S60_L40.notna()].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 612,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.9839125437994083\n",
      "mean markup: 0.436572404260097\n",
      "[4.42857162 1.54775146]\n",
      "[0.22580644 0.64609857]\n",
      "iv R2: 0.5166688960855466\n",
      "rotator coeff and p value: -0.26237635046967617 0.004209982123665291\n",
      "λ =  0 T stat = -2.741291838692067\n",
      "0.006119812434274819\n",
      "λ =  1 T stat = -6.748144215546425\n",
      "1.4974806938781804e-11\n",
      "λ =  2 T stat = -8.436819473486494\n",
      "3.2608953098670003e-17\n",
      "λ =  3 T stat = -9.273177373818525\n",
      "1.806816144773923e-20\n",
      "λ =  4 T stat = -9.753081763464547\n",
      "1.789516798317075e-22\n",
      "λ =  10 T stat = -10.697956712657877\n",
      "1.0404723417179327e-26\n",
      "λ =  20 T stat = -11.020929792512968\n",
      "3.0291382820628598e-28\n",
      "λ =  30 T stat = -11.12719637481619\n",
      "9.249919593402599e-29\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df2[df2[var].notna() & (df2.date < 2020)].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv24m',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv24m']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_24m = res2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Alternative rotators specifications: 2 quarters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 613,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/3871750942.py:13: FutureWarning: Series.dt.weekofyear and Series.dt.week have been deprecated. Please use Series.dt.isocalendar().week instead.\n",
      "  df21['w'] = df21.date.dt.week\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/3871750942.py:20: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df21 = df21a[df21.groupby(['y', 'w']).size() > 1].reset_index()\n",
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/3871750942.py:67: FutureWarning: In a future version of pandas all arguments of DataFrame.drop_duplicates except for the argument 'subset' will be keyword-only.\n",
      "  df21.drop_duplicates('date' ,'last', inplace = True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5692028798783425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/3871750942.py:79: PerformanceWarning: DataFrame is highly fragmented.  This is usually the result of calling `frame.insert` many times, which has poor performance.  Consider joining all columns at once using pd.concat(axis=1) instead. To get a de-fragmented frame, use `newframe = frame.copy()`\n",
      "  df2['vix_hat'] = np.nan\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n",
      "/Users/jasonchoi/anaconda3/lib/python3.11/site-packages/linearmodels/iv/model.py:549: MissingValueWarning: \n",
      "Inputs contain missing values. Dropping rows with missing observations.\n",
      "  super().__init__(\n"
     ]
    }
   ],
   "source": [
    "df2 = pd.read_stata('quarterly.dta')\n",
    "\n",
    "for series in ['msdluk']:\n",
    "    \n",
    "    df21 = pd.read_excel('ftse.xlsx', sheet_name = series)\n",
    "    df21.columns = ['date', series, 'n0', 'n1']\n",
    "    df21 = df21[['date', series]].copy()\n",
    "\n",
    "    df21['date'] = pd.to_datetime(df21.date)\n",
    "    df21['q'] = df21.date.dt.quarter\n",
    "    df21['y'] = df21.date.dt.year\n",
    "\n",
    "    df21['w'] = df21.date.dt.week\n",
    "\n",
    "    df21 = df21[df21[series].notna()].copy()\n",
    "\n",
    "    df21 = df21[df21.date >= '1972'].copy()\n",
    "\n",
    "    df21a = df21.set_index(['y', 'w'])\n",
    "    df21 = df21a[df21.groupby(['y', 'w']).size() > 1].reset_index()\n",
    "\n",
    "    df21a = df21.sort_values('date').groupby(['y', 'w'])[series].agg(['first', 'last']).reset_index()\n",
    "\n",
    "    df21a['ret'] = (df21a['last'] - df21a['first']) / df21a['first']\n",
    "\n",
    "    df21a['formatted_date'] = df21a.y * 1000 + df21a.w * 10 + 0\n",
    "    df21a['date'] = pd.to_datetime(df21a['formatted_date'], format='%Y%W%w')\n",
    "\n",
    "    df21a['q'] = df21a.date.dt.quarter\n",
    "\n",
    "    df21 = df21a.drop(df21a.iloc[[0, -1]].index)\n",
    "\n",
    "    tab = []\n",
    "    for year in range(df21.y.min() + 1, df21.y.max() + 1):\n",
    "        for quarter in range(1, 5):\n",
    "            if quarter == 1:\n",
    "                ind = ((df21.y == year) & (df21.q == 1)) | ((df21.y == year-1) & (df21.q == 4))\n",
    "            elif quarter == 2:\n",
    "                ind = ((df21.y == year) & (df21.q == 2)) | ((df21.y == year) & (df21.q == 1))\n",
    "            elif quarter == 3:\n",
    "                ind = ((df21.y == year) & (df21.q == 3)) | ((df21.y == year) & (df21.q == 2))\n",
    "            elif quarter == 4:\n",
    "                ind = ((df21.y == year) & (df21.q == 4)) | ((df21.y == year) & (df21.q == 3))\n",
    "            tab.append([year, quarter, df21.loc[ind].ret.std()])\n",
    "\n",
    "    df21 = pd.DataFrame(tab, columns = ['y', 'q', series + '_vol'])\n",
    "\n",
    "    df21['date'] = df21.y + (df21.q - 1) * 0.25\n",
    "    \n",
    "    df2 = df2.merge(df21[['date', series + '_vol']], on = 'date', how = 'left')\n",
    "    \n",
    "df21 = pd.read_csv('MSPIUKM.csv')\n",
    "df21.columns = ['date', 'uks']\n",
    "\n",
    "df21['uks_L'] = df21.uks.shift()\n",
    "df21['ret'] = df21.uks / df21.uks_L - 1\n",
    "\n",
    "df21.date = pd.to_datetime(df21.date)\n",
    "\n",
    "df21['q'] = df21.date.dt.quarter\n",
    "df21['y'] = df21.date.dt.year\n",
    "\n",
    "df21['ukm_vol'] = df21.ret.rolling(6).std()\n",
    "\n",
    "df21['date'] = df21.y + (df21.q - 1) * 0.25\n",
    "\n",
    "df21.drop_duplicates('date' ,'last', inplace = True)\n",
    "\n",
    "df2 = df2.merge(df21[['date', 'ukm' + '_vol']], on = 'date', how = 'left')\n",
    "    \n",
    "df21 = pd.read_csv('age_dep.csv', index_col = 0)\n",
    "df2 = df2.merge(df21, on = 'date', how = 'left')\n",
    "df2.DLdep_rat_new = np.log(df2.age_dep / df2.age_dep.shift())\n",
    "\n",
    "df21 = pd.read_excel('TVR.xlsx', sheet_name = 'Data_plots')\n",
    "\n",
    "df2 = df2.merge(df21[['date', 'VIX', 'EPU']], on = 'date', how = 'left')\n",
    "\n",
    "df2['vix_hat'] = np.nan\n",
    "\n",
    "# df2t = df2[df2.date >= 1935].copy()\n",
    "df2t = df2[df2.date >= 1926].copy()\n",
    "exog = df2t[['vol']].copy()\n",
    "\n",
    "res = IV2SLS(df2t.VIX, add_constant(exog), None, None).fit()\n",
    "\n",
    "# df2.loc[df2[df2.date >= 1935].index, 'vix_hat'] = add_constant(exog) @ res.params\n",
    "df2.loc[df2[df2.date >= 1926].index, 'vix_hat'] = add_constant(exog) @ res.params\n",
    "df2.loc[df2[df2.VIX.notna()].index, 'vix_hat'] = df2.loc[df2[df2.VIX.notna()].index, 'VIX']\n",
    "\n",
    "for series in ['msdluk_vol']:\n",
    "    df2t = df2[df2.date >= 1926].copy()\n",
    "    exog = (df2t[['ukm_vol']]).copy()\n",
    "\n",
    "    res = IV2SLS((df2t[series]), add_constant(exog), None, None).fit()\n",
    "    \n",
    "    print(res.rsquared)\n",
    "\n",
    "    df2[series + '_hat'] = np.nan\n",
    "    \n",
    "    df2.loc[(add_constant(exog) @ res.params).index, series + '_hat'] = (add_constant(exog) @ res.params)\n",
    "    df2.loc[df2[df2[series].notna()].index, series + '_hat'] = df2.loc[df2[df2[series].notna()].index, series]\n",
    "    \n",
    "\n",
    "df2 = df2[df2.spread_S60_L40.notna()].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 614,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.6417833782110827\n",
      "mean markup: 0.4074092227424438\n",
      "[1.93997563 3.34774383]\n",
      "[0.51547039 0.29870864]\n",
      "iv R2: 0.48715891434213376\n",
      "rotator coeff and p value: -0.13531816267325678 0.10259875599517716\n",
      "λ =  0 T stat = -4.3539190938647945\n",
      "1.3372507669730796e-05\n",
      "λ =  1 T stat = -5.8834307335849685\n",
      "4.0184831480605776e-09\n",
      "λ =  2 T stat = -6.36101258377572\n",
      "2.0042802193653036e-10\n",
      "λ =  3 T stat = -6.558443975262477\n",
      "5.4372126690115e-11\n",
      "λ =  4 T stat = -6.658029276356258\n",
      "2.7752350468709322e-11\n",
      "λ =  10 T stat = -6.816014084012732\n",
      "9.360112396526282e-12\n",
      "λ =  20 T stat = -6.853810335695833\n",
      "7.1908407809488856e-12\n",
      "λ =  30 T stat = -6.86364548020831\n",
      "6.71250966090512e-12\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df2[df2[var].notna() & (df2.date < 2020)].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "# dft[vb] =  np.log(dft[var])\n",
    "# vb = var\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv6m',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean())).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean())).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "# print(res2.summary)\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv6m']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "#         vb = 'vix_hat_bin'\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "#         z = dftt[vb].to_numpy()\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        # print(np.corrcoef(m1_y_dm, z_dm)[0, 1], np.corrcoef(pc_y_dm, z_dm)[0, 1])\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_6m = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 615,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_iv_bl, res_iv_resid2, res_resid,\n",
    "                  res_bin_66, res_bin_75, res_24m, res_6m], stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['lprivatepar', 'msdluk_vol_hat_bin', 'lppXvol'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 616,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "\n",
    "tabs2 = pd.DataFrame(tabs_tstat).T\n",
    "tabs2.columns = tabs2.iloc[0]\n",
    "tabs2.drop(index = 0, inplace = True)\n",
    "tabs2 = tabs2.astype(float).round(2)\n",
    "tabs2 = tabs2.T\n",
    "\n",
    "n = int(tabs2.shape[1] / 2)\n",
    "tabs3 = np.empty((tabs2.shape[0] * 2, 1 + n), dtype = object)\n",
    "tabs2 = tabs2.reset_index()\n",
    "for i in range(tabs2.shape[0]):\n",
    "    tabs3[i*2] = tabs2.iloc[i, :n+1]\n",
    "    tabs3[i*2+1, 1:] = '(' + tabs2.iloc[i, n+1:].astype(str) + ')'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.7: US public debt demand estimation: Alternative rotators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 617,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = sc.tables[0].iloc[:7, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline', 'UK Vol Residualized with detrended GDP',\n",
    "               'Non-Residualized UK Vol',\n",
    "               '66th perc. cutoff', '75th perc. cutoff', '8 quarters std dev', '2 quarters std dev']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'FVD $\\\\times$ Log(debt/gdp)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_bl.nobs), int(res_iv_resid2.nobs),\n",
    "                    int(res_resid.nobs),\n",
    "                  int(res_bin_66.nobs), int(res_bin_75.nobs), int(res_24m.nobs), int(res_6m.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity, high foreign vol', 'Demand elasticity, low foreign vol',\n",
    "                    'Markup, high foreign vol', 'Markup, low foreign vol']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 618,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', 'c', 'c', \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', 't', 't', \"t\"])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', \"t\"])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1),\n",
    "                                                           ('(5)', 1), ('(6)', 1), ('(7)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb5.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.8: Government conduct test: Alternative rotators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 619,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((19, 9), ''),\n",
    "                   columns = ['Cost elasticity', '$\\\\lambda = 0$', '$\\\\lambda = 1$', '$\\\\lambda = 2$',\n",
    "                              '$\\\\lambda = 3$', '$\\\\lambda = 4$', '$\\\\lambda = 10$', '$\\\\lambda = 20$',\n",
    "                              '$\\\\lambda = 30$'])\n",
    "tab.iloc[0, 0] = '\\\\emph{a. Baseline} \\\\vspace{.5em}'\n",
    "tab.iloc[[1, 2], 1:] = tabs3[[0, 1], 1:]\n",
    "tab.iloc[3, 0] = '\\\\emph{b. Alternative volatility based rotators} \\\\vspace{.5em}'\n",
    "tab.iloc[4:16, 1:] = tabs3[4:16, 1:]\n",
    "tab.iloc[[4, 6, 8, 10, 12, 14], 0] = ['UK Vol Residualized with detrended GDP',\n",
    "                                      'Non-Residualized UK Vol',\n",
    "               '66th perc. cutoff', '75th perc. cutoff', '8 quarters std dev', '2 quarters std dev']\n",
    "tab.iloc[16, 0] = '\\\\emph{c. Investor composition rotator} \\\\vspace{.5em}'\n",
    "tab.iloc[17:19, 1:] = tabs3[2:4, 1:]\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 620,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb6.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 621,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs1 = pd.DataFrame(tabs_fstat).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "tabs1 = tabs1.T.reset_index().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 622,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((4, 9), ''),\n",
    "                   columns = ['Cost elasticity', '$\\\\lambda = 0$', '$\\\\lambda = 1$', '$\\\\lambda = 2$',\n",
    "                              '$\\\\lambda = 3$', '$\\\\lambda = 4$', '$\\\\lambda = 10$', '$\\\\lambda = 20$',\n",
    "                              '$\\\\lambda = 30$'])\n",
    "tab.iloc[0, 0] = '\\\\emph{a. Baseline} \\\\vspace{.5em}'\n",
    "tab.iloc[1, 1:] = tabs1[0, 1:]\n",
    "tab.iloc[2, 0] = '\\\\emph{b. Investor composition} \\\\vspace{.5em}'\n",
    "tab.iloc[3, 1:] = tabs1[1, 1:]\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table B.6: F-statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 623,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb4.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demand estimation for alternative measures of spread and debt: Without rotators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 624,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs_elas = []\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "\n",
    "tabs_elas.append(['ni_iv',\n",
    "                  (-1/elas).mean(),\n",
    "                \n",
    "                  -elas.mean()])\n",
    "\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "\n",
    "\n",
    "res_iv_ni = res2\n",
    "\n",
    "### Short\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_short'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['short', -1/elas, -elas])\n",
    "\n",
    "res_short = res2\n",
    "\n",
    "### Short CD\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.spread_sa.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_sa'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['shortcd', -1/elas, -elas])\n",
    "\n",
    "res_shortcd = res2\n",
    "\n",
    "### Long\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['long', -1/elas, -elas])\n",
    "\n",
    "res_long = res2\n",
    "\n",
    "### External\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "dft['lprivatepar'] = dft['lexternal']\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft[debt].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope',  vb, 'date']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt, 'date']])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'date', 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['ex', -1/elas, -elas])\n",
    "\n",
    "res_ex = res2\n",
    "\n",
    "### trend gdp\n",
    "\n",
    "dft = df.copy()\n",
    "dft.set_index('date', inplace = True)\n",
    "cycle, trend = sm.tsa.filters.hpfilter(dft.ngdp, 1600)\n",
    "\n",
    "dft['privatepar2'] = dft.privatepar_nogdp / trend\n",
    "dft['lprivatepar2'] = np.log(dft.privatepar2)\n",
    "\n",
    "dft.reset_index(inplace = True)\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = dft[dft[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "dft['lprivatepar'] = dft['lprivatepar2']\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "                \n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['tr', -1/elas, -elas])\n",
    "\n",
    "res_tr = res2\n",
    "\n",
    "### Detrended log debt\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "dft['r_privatepar_nogdp'] = dft.privatepar_nogdp / dft.pgdp\n",
    "dft['lr_privatepar_nogdp'] = np.log(dft['r_privatepar_nogdp'])\n",
    "dft['dt_lr_privatepar_nogdp'] = sm.OLS(dft['lr_privatepar_nogdp'], sm.add_constant(dft.date)).fit().resid\n",
    "dft['lprivatepar'] = dft.dt_lr_privatepar_nogdp\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "\n",
    "tabs_elas.append(['ni_dtdebt',\n",
    "                  (-1/elas).mean(),\n",
    "                \n",
    "                  -elas.mean()])\n",
    "\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "\n",
    "res_dtdebt = res2\n",
    "\n",
    "###\n",
    "\n",
    "sc = summary_col([res_iv_ni, res_short, res_shortcd, res_long, res_ex, res_tr,\n",
    "                  res_dtdebt], stars = True, float_format = '%.2f',\n",
    "                 regressor_order = ['lprivatepar', vb])\n",
    "\n",
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 625,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = sc.tables[0].iloc[:5, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline', 'Short Mat. (CP)', 'Short Mat. (CD)', 'Long Mat.',\n",
    "               'External Debt', 'Debt to Trend GDP', 'Detrended Log of Real Debt']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_ni.nobs), int(res_short.nobs), int(res_shortcd.nobs),\n",
    "                    int(res_long.nobs), int(res_ex.nobs), int(res_tr.nobs), int(res_dtdebt.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity', 'Markup']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.12: US public debt demand estimation without rotators: Alternative measures of spreads and debt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 626,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', 'c', 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', 't', 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1),\n",
    "                                                           ('(5)', 1), ('(6)', 1), ('(7)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb9.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demand estimation for different controls: Without rotators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 627,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs_elas = []\n",
    "\n",
    "### baseline\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['base', -1/elas, -elas])\n",
    "\n",
    "res_iv_ni = res2\n",
    "\n",
    "### Deposits\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[(dft.date >= 1935) & (dft[var].notna())\n",
    "          & dft['tol_dep_interp'].notna()].copy()\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v), 'slope',\n",
    "                                                           'DLdep_rat_new', vb, 'vix_hat', 'tol_dep_interp']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat', 'tol_dep_interp']])\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat', 'tol_dep_interp']])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['tol_dep', -1/elas, -elas])\n",
    "\n",
    "res_tol_dep = res2\n",
    "\n",
    "### BAA/AAA spread + Corp. default\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.ldebtuk2.notna())].copy()\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "dft = dft[dft['CorpDefRate'].notna()].copy()\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v), 'slope',\n",
    "                                                           'DLdep_rat_new', vb, 'vix_hat',\n",
    "                                                       'BAA_AAA_spread', 'CorpDefRate']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[ 'slope', vb, debt, 'vix_hat',\n",
    "                                         'BAA_AAA_spread', 'CorpDefRate']])\n",
    "\n",
    "                instr = add_constant(dft[[ 'slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat',\n",
    "                                          'BAA_AAA_spread', 'CorpDefRate']])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['bacorp', -1/elas, -elas])\n",
    "\n",
    "res_bacorp = res2\n",
    "\n",
    "\n",
    "### Less controls\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['lc', -1/elas, -elas])\n",
    "\n",
    "res_lc = res2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 628,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_iv_ni, res_tol_dep, res_bacorp, res_lc], stars = True, float_format = '%.2f', \n",
    "                regressor_order = ['lprivatepar', vb])\n",
    "\n",
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.15: US public debt demand estimation without rotators: Different controls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 629,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = sc.tables[0].iloc[:5, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline', 'Control for Deposits',\n",
    "               'Control for BAA/AAA spread + Corp. Default',\n",
    "               'No controls']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_ni.nobs), int(res_tol_dep.nobs), int(res_bacorp.nobs),\n",
    "                    int(res_lc.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity', 'Markup']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb12.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demand estimation for additional robustness: without rotators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 630,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabs_elas = []\n",
    "\n",
    "### baseline\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['base', -1/elas, -elas])\n",
    "\n",
    "res_iv_ni = res2\n",
    "\n",
    "\n",
    "### ZLB\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna()) & (dft.ZLB == 0)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['zlb', -1/elas, -elas])\n",
    "\n",
    "res_zlb = res2\n",
    "\n",
    "### Post 45\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1945) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['p45', -1/elas, -elas])\n",
    "\n",
    "res_p45 = res2\n",
    "\n",
    "\n",
    "### Pre 08\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.date < 2008) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['p08', -1/elas, -elas])\n",
    "\n",
    "res_p08 = res2\n",
    "\n",
    "### Post 72\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1972) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new',   'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['p72', -1/elas, -elas])\n",
    "\n",
    "res_p72 = res2\n",
    "\n",
    "\n",
    "### Ramey\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                             'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'rameynewsgdp_%i_%i'%(xx, v)]])\n",
    "                \n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['ramey', -1/elas, -elas])\n",
    "\n",
    "res_ramey = res2\n",
    "\n",
    "### Deprat only\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "depvar = dft[[spread]]\n",
    "\n",
    "exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'DLdep_rat_new']])\n",
    "\n",
    "res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['dep', -1/elas, -elas])\n",
    "\n",
    "res_dep = res2\n",
    "\n",
    "### BP\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    " \n",
    " \n",
    " \n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'bpshock_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['bpshock_%i_%i'%(xx, v),\n",
    "                                                             'slope', 'vix_hat', vb]]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[[  'slope', 'vix_hat', vb, debt]])\n",
    "\n",
    "                instr = add_constant(dft[[  'slope', 'vix_hat', vb, 'bpshock_%i_%i'%(xx, v)]])\n",
    "                \n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "                \n",
    "elas = (res2.params[debt]) / dft[spread].mean()\n",
    "tabs_elas.append(['bp', -1/elas, -elas])\n",
    "\n",
    "res_bp = res2\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.18: US public debt demand estimation without rotators: Additional robustness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 631,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = summary_col([res_iv_ni, res_zlb, res_p45, res_p08, res_p72,\n",
    "                  res_ramey, res_dep, res_bp], stars = True, float_format = '%.2f', \n",
    "                regressor_order = ['lprivatepar', vb])\n",
    "\n",
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "\n",
    "tab = sc.tables[0].iloc[:5, :].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'Baseline',\n",
    "               'No ZLB', 'Post War', 'Pre GFC', 'High External Debt', 'Military', 'Dependency', 'BP shock']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '', 'Observations']\n",
    "tab.iloc[-1, 1:] = [int(res_iv_ni.nobs),\n",
    "                    int(res_zlb.nobs), int(res_p45.nobs), int(res_p08.nobs), int(res_p72.nobs),\n",
    "                  int(res_ramey.nobs), int(res_dep.nobs), int(res_bp.nobs)]\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity', 'Markup']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c', 'c', 'c', \"c\", \"c\"])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't', 't', 't', \"t\", 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't', 't', 't', \"t\", 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1),\n",
    "                                                           ('(5)', 1), ('(6)', 1), ('(7)', 1), ('(8)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tabb12_b.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table B.3: Demand rotator correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 632,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('gov_exp.csv')\n",
    "df1['date'] = pd.to_datetime(df1.date)\n",
    "df1['date'] = df1.date.dt.year + (df1.date.dt.quarter - 1) / 4\n",
    "dft = df.merge(df1, on = 'date', how = 'left')\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = dft[dft[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.nanquantile(dft[var], 0.5)) + 0\n",
    "\n",
    "dft['fed_gdp'] = dft.fed / dft.ngdp\n",
    "\n",
    "dft[['date', 'msdluk_vol_hat', 'fed_gdp']].to_csv('fed_gdp.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 633,
   "metadata": {},
   "outputs": [],
   "source": [
    "rlist = [(1937.25, 1938.25), (1945, 1945.75), (1948.75, 1949.75), (1953.25, 1954.25), (1957.5, 1958.25), (1960.25, 1961),\n",
    "             (1969.75, 1970.75), (1973.75, 1975), (1980, 1980.5), (1981.5, 1982.75), (1990.5, 1991), (2001, 2001.75), \n",
    "             (2007.75, 2009.25), (2019.75, 2020.25)]\n",
    "recess = []\n",
    "for i in rlist:\n",
    "    recess.append(np.arange(i[0], i[1] + 0.25, 0.25))\n",
    "    \n",
    "recess = np.hstack(recess)\n",
    "\n",
    "df['recess'] = df.date.isin(recess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 634,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_excel('Tax rate Data_v3.xlsx', sheet_name = 'Data')\n",
    "\n",
    "df1 = df1[df1.country == 'United States'].copy()\n",
    "\n",
    "df1 =  df1[['year', 'corporate_tr', 'individual_tr']].copy()\n",
    "\n",
    "df1['dcorp_tr'] = df1.corporate_tr - df1.corporate_tr.shift()\n",
    "df1['dind_tr'] = df1.individual_tr - df1.individual_tr.shift()\n",
    "df1['dlcorp_tr'] = np.log(df1.corporate_tr / df1.corporate_tr.shift())\n",
    "df1['dlind_tr'] = np.log(df1.individual_tr / df1.individual_tr.shift())\n",
    "\n",
    "dft = df.copy()\n",
    "dft['year'] = np.floor(dft.date)\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = dft[dft[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "dft[var + '_og'] = dft[var].copy()\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.nanquantile(dft[var], 0.5)) + 0\n",
    "\n",
    "df2 = dft.groupby(['year'])[['msdluk_vol_hat_og', 'msdluk_vol_hat', 'msdluk_vol_hat_bin']].mean().reset_index()\n",
    "\n",
    "df2 = df2.merge(df1, on = 'year', how = 'left')\n",
    "\n",
    "df1 = pd.read_csv('fed_deficit.csv')\n",
    "df1.columns = ['year', 'fed_surplus']\n",
    "df1.year = pd.to_datetime(df1.year).dt.year\n",
    "\n",
    "df2 = df2.merge(df1, on = 'year', how = 'left')\n",
    "df2['fed_deficit'] = -df2.fed_surplus\n",
    "\n",
    "\n",
    "corr = df2[['msdluk_vol_hat_og', 'msdluk_vol_hat', 'msdluk_vol_hat_bin',\n",
    "           'fed_deficit']].corr().iloc[:3, 3:].round(2)\n",
    "corr['fed_deficit_t'] = 0\n",
    "for ii, var in enumerate(['msdluk_vol_hat_og', 'msdluk_vol_hat', 'msdluk_vol_hat_bin']):\n",
    "    n = df2[[var, 'fed_deficit']].dropna().shape[0]\n",
    "    cc = corr.iloc[ii, 0]\n",
    "    tstat = cc * (n-2)**0.5 / (1-cc**2)**0.5\n",
    "    corr.loc[var, 'fed_deficit_t'] = '(' + str(np.round(tstat, 2)) + ')'\n",
    "\n",
    "tlist1 = [corr]\n",
    "\n",
    "df1 = pd.read_csv('gov_exp.csv')\n",
    "df1['date'] = pd.to_datetime(df1.date)\n",
    "df1['date'] = df1.date.dt.year + (df1.date.dt.quarter - 1) / 4\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "dft[var + '_og'] = dft[var].copy()\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "\n",
    "df2 = dft.merge(df1, on = 'date', how = 'left')\n",
    "df2['Ddebt'] = df2.privatepar - df2.privatepar.shift(4)\n",
    "\n",
    "df1 = pd.read_csv('GDPDEF.csv')\n",
    "df1.columns = ['date', 'gdpd']\n",
    "df1['date'] = pd.to_datetime(df1.date)\n",
    "df1['date'] = df1.date.dt.year + (df1.date.dt.quarter - 1) / 4\n",
    "df2 = df2.merge(df1, on = 'date', how = 'left')\n",
    "\n",
    "df1 = pd.read_csv('gsb.csv')\n",
    "df1.columns = ['date', 'gsb']\n",
    "df1['date'] = pd.to_datetime(df1.date)\n",
    "df1['date'] = df1.date.dt.year + (df1.date.dt.quarter - 1) / 4\n",
    "df2 = df2.merge(df1, on = 'date', how = 'left')\n",
    "\n",
    "df2['fed_r'] = df2.fed / df2.gdpd * 100\n",
    "df2['fed_growth'] = (df2.fed - df2.fed.shift(4)) / df2.fed\n",
    "df2['fed_growth_r'] = (df2.fed_r - df2.fed_r.shift(4)) / df2.fed_r\n",
    "df2['gsb_r'] = df2.gsb / df2.gdpd * 100\n",
    "df2['gsb_growth'] = (df2.gsb - df2.gsb.shift(4)) / df2.gsb\n",
    "df2['gsb_growth_r'] = (df2.gsb_r - df2.gsb_r.shift(4)) / df2.gsb_r\n",
    "\n",
    "df2['fed_gdp'] = df2.fed / df2.ngdp\n",
    "df2['gsb_gdp'] = df2.gsb / df2.ngdp\n",
    "\n",
    "corr = df2[['msdluk_vol_hat_og', 'msdluk_vol_hat', 'msdluk_vol_hat_bin',\n",
    "               'fed_gdp', 'gsb_gdp', 'Ddebt', 'recess', 'bpshock_5_40']].corr().iloc[:3, 3:].round(2)\n",
    "for var2 in ['fed_gdp', 'gsb_gdp', 'Ddebt', 'recess', 'bpshock_5_40']:\n",
    "    corr[var2 + '_t'] = 0\n",
    "\n",
    "for ii, var in enumerate(['msdluk_vol_hat_og', 'msdluk_vol_hat', 'msdluk_vol_hat_bin']):\n",
    "    for ii2, var2 in enumerate(['fed_gdp', 'gsb_gdp', 'Ddebt', 'recess', 'bpshock_5_40']):\n",
    "        n = df2[[var, var2]].dropna().shape[0]\n",
    "        cc = corr.loc[var, var2]\n",
    "        tstat = cc * (n-2)**0.5 / (1-cc**2)**0.5\n",
    "        corr.loc[var, var2 + '_t'] = '(' + str(np.round(tstat, 2)) + ')'\n",
    "\n",
    "tlist2 = [corr]\n",
    "\n",
    "tlist = []\n",
    "for i in range(1):\n",
    "    tlist.append(tlist1[i].merge(tlist2[i], left_index = True, right_index = True))\n",
    "\n",
    "tlist = pd.concat(tlist)\n",
    "tlist.columns = ['Gov. deficit', 'Gov. deficit_t', 'Gov. spending/GDP',\n",
    "                 'Gov. benefits transfer/GDP', 'Change in stock of treasuries', 'Recession indicator',\n",
    "                 'Blanchard-Perotti shock',\n",
    "                 'Gov. spending/GDP_t', 'Gov. benefits transfer/GDP_t', 'Change in stock of treasuries_t',\n",
    "                 'Recession indicator_t', 'Blanchard-Perotti shock_t']\n",
    "tt1 = tlist[['Gov. spending/GDP', 'Gov. benefits transfer/GDP', 'Blanchard-Perotti shock', 'Recession indicator', \n",
    "                'Gov. deficit', 'Change in stock of treasuries']].values\n",
    "tt2 = tlist[['Gov. spending/GDP_t', 'Gov. benefits transfer/GDP_t', 'Blanchard-Perotti shock_t', 'Recession indicator_t',\n",
    "                'Gov. deficit_t', 'Change in stock of treasuries_t']].values\n",
    "\n",
    "tab = pd.DataFrame(np.full((6, 5), ''),\n",
    "                   columns = ['', 'Gov. spending/GDP', 'Gov. benefits transfer/GDP','Blanchard-Perotti shock',\n",
    "                              'Recession indicator'])\n",
    "tab.iloc[[0, 2, 4], 1:] = tt1[0:3, :-2]\n",
    "tab.iloc[[0, 2, 4], 0] = ['UK volatility, non-residualized', 'UK volatility, residualized', 'UK volatility indicator']\n",
    "tab.iloc[[1, 3, 5], 1:] = tt2[0:3, :-2]\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb2.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table B.4: Fiscal supply instrument correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 635,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "dft[var + '_og'] = dft[var].copy()\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "\n",
    "corr = dft[['DLdep_rat_new', 'rameynewsgdp_5_40', 'bpshock_5_40', 'msdluk_vol_hat',\n",
    "     'VIX', 'rgdp_grate']].corr().iloc[:3, 3:].round(2)\n",
    "\n",
    "for var2 in ['msdluk_vol_hat', 'VIX', 'rgdp_grate']:\n",
    "    corr[var2 + '_t'] = 0\n",
    "\n",
    "for ii, var in enumerate(['DLdep_rat_new', 'rameynewsgdp_5_40', 'bpshock_5_40']):\n",
    "    for ii2, var2 in enumerate(['msdluk_vol_hat', 'VIX', 'rgdp_grate']):\n",
    "        n = dft[[var, var2]].dropna().shape[0]\n",
    "        cc = corr.loc[var, var2]\n",
    "        tstat = cc * (n-2)**0.5 / (1-cc**2)**0.5\n",
    "        corr.loc[var, var2 + '_t'] = '(' + str(np.round(tstat, 2)) + ')'\n",
    "        \n",
    "corr.columns = ['UK volatility, residualized', 'VIX', 'GDP growth',\n",
    "                'UK volatility, residualized_t', 'VIX_t', 'GDP growth_t']\n",
    "tt1 = corr[['UK volatility, residualized', 'VIX', 'GDP growth']].values\n",
    "tt2 = corr[['UK volatility, residualized_t', 'VIX_t', 'GDP growth_t']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 636,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab = pd.DataFrame(np.full((6, 4), ''),\n",
    "                   columns = ['', 'UK volatility, residualized', 'VIX', 'GDP growth'])\n",
    "tab.iloc[[0, 2, 4], 1:] = tt1[:, :]\n",
    "tab.iloc[[0, 2, 4], 0] = ['$\\\\Delta$ Log dependency ratio', 'Military news shock', 'Blanchard-Perotti shock']\n",
    "tab.iloc[[1, 3, 5], 1:] = tt2[:, :]\n",
    "tab.iloc[:, -1] = tab.iloc[:, -1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "tex_file = open('tabb3.tex', 'w')\n",
    "tex_file.write(latextable.draw_latex(table, use_booktabs = True))\n",
    "tex_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table C.1: Debt-to-GDP and Volatility Comovement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 637,
   "metadata": {},
   "outputs": [],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[dft.date>=1935].copy()\n",
    "\n",
    "var = 'vix_hat_ma'\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.nanquantile(dft[var], 0.5)) + 0\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "\n",
    "dft['Ddebt'] = dft.privatepar - dft.privatepar.shift(4)\n",
    "\n",
    "tab = dft[['privatepar', 'msdluk_vol_hat',\n",
    "           'msdluk_vol_hat_bin']].corr().iloc[:1, 1:].round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 638,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/ql/2phccyg93dv79h6f6mm97qq80000gn/T/ipykernel_33730/2231412350.py:4: FutureWarning: In future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n",
      "  tab.to_latex(buf = 'tabc2.tex', escape = False, column_format = 'lcc')\n"
     ]
    }
   ],
   "source": [
    "tab.columns = ['UK volatility', 'UK vol indicator']\n",
    "tab.index = ['Debt to GDP']\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "tab.to_latex(buf = 'tabc2.tex', escape = False, column_format = 'lcc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure 2: Safe assets demand in times of high and low volatility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 639,
   "metadata": {},
   "outputs": [],
   "source": [
    "### kvj\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = df[df.date>= 1935].copy()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "dft[['date', 'privatepar', 'spread_S60_L40', 'msdluk_vol_hat_bin']].to_csv('kvj_scatter_uk_resid2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 640,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>     <td>spread_S60_L40</td>  <th>  R-squared:         </th> <td>   0.442</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.437</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   88.36</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Tue, 05 Sep 2023</td> <th>  Prob (F-statistic):</th> <td>3.83e-42</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>14:58:35</td>     <th>  Log-Likelihood:    </th> <td>  7.6360</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   339</td>      <th>  AIC:               </th> <td>  -7.272</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   335</td>      <th>  BIC:               </th> <td>   8.032</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     3</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "           <td></td>             <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>              <td>    0.3038</td> <td>    0.043</td> <td>    6.992</td> <td> 0.000</td> <td>    0.218</td> <td>    0.389</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>lprivatepar</th>        <td>   -0.2503</td> <td>    0.047</td> <td>   -5.359</td> <td> 0.000</td> <td>   -0.342</td> <td>   -0.158</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>msdluk_vol_hat_bin</th> <td>   -0.2925</td> <td>    0.073</td> <td>   -3.990</td> <td> 0.000</td> <td>   -0.437</td> <td>   -0.148</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>lppXvol</th>            <td>   -0.3853</td> <td>    0.068</td> <td>   -5.664</td> <td> 0.000</td> <td>   -0.519</td> <td>   -0.252</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>142.549</td> <th>  Durbin-Watson:     </th> <td>   0.590</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td> 978.739</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.596</td>  <th>  Prob(JB):          </th> <td>2.95e-213</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td>10.687</td>  <th>  Cond. No.          </th> <td>    14.2</td> \n",
       "</tr>\n",
       "</table><br/><br/>Notes:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}        & spread\\_S60\\_L40 & \\textbf{  R-squared:         } &     0.442   \\\\\n",
       "\\textbf{Model:}                &       OLS        & \\textbf{  Adj. R-squared:    } &     0.437   \\\\\n",
       "\\textbf{Method:}               &  Least Squares   & \\textbf{  F-statistic:       } &     88.36   \\\\\n",
       "\\textbf{Date:}                 & Tue, 05 Sep 2023 & \\textbf{  Prob (F-statistic):} &  3.83e-42   \\\\\n",
       "\\textbf{Time:}                 &     14:58:35     & \\textbf{  Log-Likelihood:    } &    7.6360   \\\\\n",
       "\\textbf{No. Observations:}     &         339      & \\textbf{  AIC:               } &    -7.272   \\\\\n",
       "\\textbf{Df Residuals:}         &         335      & \\textbf{  BIC:               } &     8.032   \\\\\n",
       "\\textbf{Df Model:}             &           3      & \\textbf{                     } &             \\\\\n",
       "\\textbf{Covariance Type:}      &    nonrobust     & \\textbf{                     } &             \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                               & \\textbf{coef} & \\textbf{std err} & \\textbf{t} & \\textbf{P$> |$t$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{const}                 &       0.3038  &        0.043     &     6.992  &         0.000        &        0.218    &        0.389     \\\\\n",
       "\\textbf{lprivatepar}           &      -0.2503  &        0.047     &    -5.359  &         0.000        &       -0.342    &       -0.158     \\\\\n",
       "\\textbf{msdluk\\_vol\\_hat\\_bin} &      -0.2925  &        0.073     &    -3.990  &         0.000        &       -0.437    &       -0.148     \\\\\n",
       "\\textbf{lppXvol}               &      -0.3853  &        0.068     &    -5.664  &         0.000        &       -0.519    &       -0.252     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lclc}\n",
       "\\textbf{Omnibus:}       & 142.549 & \\textbf{  Durbin-Watson:     } &     0.590  \\\\\n",
       "\\textbf{Prob(Omnibus):} &   0.000 & \\textbf{  Jarque-Bera (JB):  } &   978.739  \\\\\n",
       "\\textbf{Skew:}          &   1.596 & \\textbf{  Prob(JB):          } & 2.95e-213  \\\\\n",
       "\\textbf{Kurtosis:}      &  10.687 & \\textbf{  Cond. No.          } &      14.2  \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{OLS Regression Results}\n",
       "\\end{center}\n",
       "\n",
       "Notes: \\newline\n",
       " [1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:         spread_S60_L40   R-squared:                       0.442\n",
       "Model:                            OLS   Adj. R-squared:                  0.437\n",
       "Method:                 Least Squares   F-statistic:                     88.36\n",
       "Date:                Tue, 05 Sep 2023   Prob (F-statistic):           3.83e-42\n",
       "Time:                        14:58:35   Log-Likelihood:                 7.6360\n",
       "No. Observations:                 339   AIC:                            -7.272\n",
       "Df Residuals:                     335   BIC:                             8.032\n",
       "Df Model:                           3                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "======================================================================================\n",
       "                         coef    std err          t      P>|t|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------------\n",
       "const                  0.3038      0.043      6.992      0.000       0.218       0.389\n",
       "lprivatepar           -0.2503      0.047     -5.359      0.000      -0.342      -0.158\n",
       "msdluk_vol_hat_bin    -0.2925      0.073     -3.990      0.000      -0.437      -0.148\n",
       "lppXvol               -0.3853      0.068     -5.664      0.000      -0.519      -0.252\n",
       "==============================================================================\n",
       "Omnibus:                      142.549   Durbin-Watson:                   0.590\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              978.739\n",
       "Skew:                           1.596   Prob(JB):                    2.95e-213\n",
       "Kurtosis:                      10.687   Cond. No.                         14.2\n",
       "==============================================================================\n",
       "\n",
       "Notes:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 640,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = dft.spread_S60_L40\n",
    "X = sm.add_constant(dft[['lprivatepar', vb]])\n",
    "X['lppXvol'] = X['lprivatepar'] * X[vb]\n",
    "sm.OLS(Y, X, missing = 'drop').fit().summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 641,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>     <td>spread_S60_L40</td>  <th>  R-squared:         </th> <td>   0.370</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.368</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   197.8</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Tue, 05 Sep 2023</td> <th>  Prob (F-statistic):</th> <td>1.13e-35</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>14:58:35</td>     <th>  Log-Likelihood:    </th> <td> -12.889</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   339</td>      <th>  AIC:               </th> <td>   29.78</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   337</td>      <th>  BIC:               </th> <td>   37.43</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     1</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "       <td></td>          <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>       <td>    0.1567</td> <td>    0.036</td> <td>    4.357</td> <td> 0.000</td> <td>    0.086</td> <td>    0.227</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>lprivatepar</th> <td>   -0.4724</td> <td>    0.034</td> <td>  -14.065</td> <td> 0.000</td> <td>   -0.538</td> <td>   -0.406</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>155.705</td> <th>  Durbin-Watson:     </th> <td>   0.488</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>1178.487</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.744</td>  <th>  Prob(JB):          </th> <td>1.24e-256</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td>11.442</td>  <th>  Cond. No.          </th> <td>    5.07</td> \n",
       "</tr>\n",
       "</table><br/><br/>Notes:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/latex": [
       "\\begin{center}\n",
       "\\begin{tabular}{lclc}\n",
       "\\toprule\n",
       "\\textbf{Dep. Variable:}    & spread\\_S60\\_L40 & \\textbf{  R-squared:         } &     0.370   \\\\\n",
       "\\textbf{Model:}            &       OLS        & \\textbf{  Adj. R-squared:    } &     0.368   \\\\\n",
       "\\textbf{Method:}           &  Least Squares   & \\textbf{  F-statistic:       } &     197.8   \\\\\n",
       "\\textbf{Date:}             & Tue, 05 Sep 2023 & \\textbf{  Prob (F-statistic):} &  1.13e-35   \\\\\n",
       "\\textbf{Time:}             &     14:58:35     & \\textbf{  Log-Likelihood:    } &   -12.889   \\\\\n",
       "\\textbf{No. Observations:} &         339      & \\textbf{  AIC:               } &     29.78   \\\\\n",
       "\\textbf{Df Residuals:}     &         337      & \\textbf{  BIC:               } &     37.43   \\\\\n",
       "\\textbf{Df Model:}         &           1      & \\textbf{                     } &             \\\\\n",
       "\\textbf{Covariance Type:}  &    nonrobust     & \\textbf{                     } &             \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lcccccc}\n",
       "                     & \\textbf{coef} & \\textbf{std err} & \\textbf{t} & \\textbf{P$> |$t$|$} & \\textbf{[0.025} & \\textbf{0.975]}  \\\\\n",
       "\\midrule\n",
       "\\textbf{const}       &       0.1567  &        0.036     &     4.357  &         0.000        &        0.086    &        0.227     \\\\\n",
       "\\textbf{lprivatepar} &      -0.4724  &        0.034     &   -14.065  &         0.000        &       -0.538    &       -0.406     \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "\\begin{tabular}{lclc}\n",
       "\\textbf{Omnibus:}       & 155.705 & \\textbf{  Durbin-Watson:     } &     0.488  \\\\\n",
       "\\textbf{Prob(Omnibus):} &   0.000 & \\textbf{  Jarque-Bera (JB):  } &  1178.487  \\\\\n",
       "\\textbf{Skew:}          &   1.744 & \\textbf{  Prob(JB):          } & 1.24e-256  \\\\\n",
       "\\textbf{Kurtosis:}      &  11.442 & \\textbf{  Cond. No.          } &      5.07  \\\\\n",
       "\\bottomrule\n",
       "\\end{tabular}\n",
       "%\\caption{OLS Regression Results}\n",
       "\\end{center}\n",
       "\n",
       "Notes: \\newline\n",
       " [1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:         spread_S60_L40   R-squared:                       0.370\n",
       "Model:                            OLS   Adj. R-squared:                  0.368\n",
       "Method:                 Least Squares   F-statistic:                     197.8\n",
       "Date:                Tue, 05 Sep 2023   Prob (F-statistic):           1.13e-35\n",
       "Time:                        14:58:35   Log-Likelihood:                -12.889\n",
       "No. Observations:                 339   AIC:                             29.78\n",
       "Df Residuals:                     337   BIC:                             37.43\n",
       "Df Model:                           1                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "===============================================================================\n",
       "                  coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-------------------------------------------------------------------------------\n",
       "const           0.1567      0.036      4.357      0.000       0.086       0.227\n",
       "lprivatepar    -0.4724      0.034    -14.065      0.000      -0.538      -0.406\n",
       "==============================================================================\n",
       "Omnibus:                      155.705   Durbin-Watson:                   0.488\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             1178.487\n",
       "Skew:                           1.744   Prob(JB):                    1.24e-256\n",
       "Kurtosis:                      11.442   Cond. No.                         5.07\n",
       "==============================================================================\n",
       "\n",
       "Notes:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 641,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y = dft.spread_S60_L40\n",
    "X = sm.add_constant(dft[['lprivatepar']])\n",
    "sm.OLS(Y, X, missing = 'drop').fit().summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Figure B.4: Inferring US government conduct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 642,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 2.797608376985866\n",
      "mean markup: 0.40772897074100223\n",
      "[3.78396329 1.81705555]\n",
      "[0.26427318 0.55034091]\n",
      "iv R2: 0.49412192728885596\n",
      "rotator coeff and p value: -0.1785839003289844 0.033014321558760544\n"
     ]
    }
   ],
   "source": [
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "\n",
    "                depvar = dft[[spread]]\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb]) / dft[spread].mean()\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "tab = []\n",
    "\n",
    "year = 1935 \n",
    "    \n",
    "elash = (res2.params[debt] + res2.params.lppXvol * 1) / dft[spread].mean()\n",
    "elasl = (res2.params[debt] + res2.params.lppXvol * 0) / dft[spread].mean()\n",
    "\n",
    "dftt = dft[(dft.date >= year)].copy()\n",
    "\n",
    "mean_debth = dftt[dftt[vb]== 1]['privatepar'].mean()\n",
    "mean_spreadh = dftt[dftt[vb]== 1][spread].mean()\n",
    "\n",
    "mean_debtl = dftt[dftt[vb]== 0]['privatepar'].mean()\n",
    "mean_spreadl = dftt[dftt[vb]== 0][spread].mean()\n",
    "\n",
    "ϵ_list = []\n",
    "\n",
    "for λ in range(31):\n",
    "    λf = (mean_spreadh - mean_spreadh * elash * (mean_spreadh - mean_spreadl) / (mean_spreadh * elash\n",
    "                                                                                 - mean_spreadl * elasl)) \\\n",
    "        / (mean_debth**λ - mean_spreadh * elash * (mean_debth**λ - mean_debtl**λ) / (mean_spreadh * elash\n",
    "                                                                                     - mean_spreadl * elasl))\n",
    "    tab.append([λ, λf, (λf * mean_debth**λ - mean_spreadh) / (mean_spreadh * elash)])\n",
    "\n",
    "    ϵ_list.append((λf * dftt['privatepar']**λ - dftt[spread]) / (dftt[spread] * elas))\n",
    "        \n",
    "pd.DataFrame(tab, columns = ['lambda', 'lambda_f', 'conduct']).to_csv('conduct_params.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B.4.3 Other Robustness Analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean elas: 1.9653621566514243\n",
      "mean markup: 0.54879935497588\n",
      "[2.49786709 1.4359896 ]\n",
      "[0.40034156 0.69638387]\n",
      "iv R2: 0.5159499201528217\n",
      "rotator coeff and p value: -0.2960423155745588 0.00053687628849691\n",
      "λ =  0 T stat = -4.218255855650993\n",
      "2.461993589367185e-05\n",
      "λ =  1 T stat = -6.8310836827274874\n",
      "8.427555160178961e-12\n",
      "λ =  2 T stat = -7.424793151022326\n",
      "1.1295630099089676e-13\n",
      "λ =  3 T stat = -7.649432241535665\n",
      "2.0186861199015774e-14\n",
      "λ =  4 T stat = -7.761668324970228\n",
      "8.381937623051659e-15\n",
      "λ =  10 T stat = -7.951818500443192\n",
      "1.837933918083236e-15\n",
      "λ =  20 T stat = -8.008145662556606\n",
      "1.164509292874032e-15\n",
      "λ =  30 T stat = -8.025729034817964\n",
      "1.0092476911933152e-15\n",
      "mean elas: 2.782194791438263\n",
      "mean markup: 0.3811491516510534\n",
      "[3.44879244 2.11951831]\n",
      "[0.28995656 0.47180531]\n",
      "iv R2: 0.4924123066772125\n",
      "rotator coeff and p value: -0.18184875150860397 0.10398116628262696\n",
      "λ =  0 T stat = -4.7504689243012965\n",
      "2.0294548182367046e-06\n",
      "λ =  1 T stat = -6.543117220313994\n",
      "6.024956763377467e-11\n",
      "λ =  2 T stat = -7.373341864500466\n",
      "1.664030450513397e-13\n",
      "λ =  3 T stat = -7.827849296046189\n",
      "4.962863169129386e-15\n",
      "λ =  4 T stat = -8.108429521096877\n",
      "5.127819337524611e-16\n",
      "λ =  10 T stat = -8.72143662425021\n",
      "2.746939733853756e-18\n",
      "λ =  20 T stat = -8.955973633195171\n",
      "3.367504773078708e-19\n",
      "λ =  30 T stat = -9.036725905812329\n",
      "1.614345194668446e-19\n",
      "mean elas: 1.8783049333773978\n",
      "mean markup: 0.5323949174758811\n",
      "mean elas: 2.646144452727901\n",
      "mean markup: 0.3779083182587041\n"
     ]
    }
   ],
   "source": [
    "# Log on log elasticity estimation\n",
    "\n",
    "tabs_elas = []\n",
    "tabs_tstat = []\n",
    "\n",
    "# Baseline\n",
    "\n",
    "# OLS\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "depvar = np.log(dft[spread])\n",
    "\n",
    "exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "res2 = sm.OLS(depvar, exog).fit()\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb])\n",
    "\n",
    "tabs_elas.append(['bl_ols',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) )).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) )).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) )).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) )).mean()])\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_ols']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[[debt]])[['const']].to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_ols_bl = res2\n",
    "\n",
    "# IV\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "\n",
    "                depvar = np.log(dft[[spread]])\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                exog['lppXvol'] = exog[debt] * exog[vb]\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['newsXvol'] = dft['rameynewsgdp_%i_%i'%(xx, v)] * dft[vb]\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                instr['depratXvol'] = dft.DLdep_rat_new * dft[vb]\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt] + res2.params.lppXvol * exog[vb])\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "print(-1/elas.unique())\n",
    "print(-elas.unique())\n",
    "\n",
    "tabs_elas.append(['bl_iv',\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 1) )).mean(),\n",
    "                  (-1/((res2.params[debt] + res2.params.lppXvol * 0) )).mean(),\n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 1) )).mean(), \n",
    "                  (-((res2.params[debt] + res2.params.lppXvol * 0) )).mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "print('iv R2:', res2.rsquared)\n",
    "print('rotator coeff and p value:', res2.params.lppXvol, res2.pvalues.lppXvol)\n",
    "\n",
    "for year in [1926]:\n",
    "\n",
    "    dftt = dft[(dft.date >= year)].copy()\n",
    "    \n",
    "    tlist1 = ['bl_iv']\n",
    "    tlist2 = []\n",
    "\n",
    "    for λ in [0, 1, 2, 3, 4, 10, 20, 30]:\n",
    "\n",
    "        m1_y = np.log((dftt[spread] + dftt.elas * dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "        pc_y = np.log((dftt[spread]).to_numpy()) - λ * dftt[debt]\n",
    "\n",
    "        z = dftt[var].to_numpy()\n",
    "\n",
    "        XX = add_constant(dftt[['rameynewsgdp_%i_%i'%(startlag, endlag), 'DLdep_rat_new']]).to_numpy()\n",
    "\n",
    "        proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T \n",
    "        m1_y_dm = m1_y - proj @ m1_y\n",
    "        pc_y_dm = pc_y - proj @ pc_y\n",
    "\n",
    "        z_dm = z - proj @ z\n",
    "\n",
    "        g_pc = (z_dm * pc_y_dm).mean()\n",
    "        g_m1 = (z_dm * m1_y_dm).mean()\n",
    "\n",
    "        W = 1 / (z_dm * z_dm).mean()\n",
    "        Q_pc = g_pc**2 * W\n",
    "        Q_m1 = g_m1**2 * W\n",
    "\n",
    "        ψ_pc = W**0.5 * (z_dm * pc_y_dm - g_pc) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_pc\n",
    "        ψ_m1 = W**0.5 * (z_dm * m1_y_dm - g_m1) - 0.5 * W**1.5 * (z_dm**2 - 1/W) * g_m1\n",
    "\n",
    "        ### m1 is 1 ; pc is 2\n",
    "        V11 = (ψ_m1 * ψ_m1).mean()\n",
    "        V22 = (ψ_pc * ψ_pc).mean()\n",
    "        V12 = (ψ_m1 * ψ_pc).mean()\n",
    "        σ = (4*(g_m1**2 * W * V11 + g_pc**2 * W * V22 - 2 * g_m1 * g_pc * W * V12))**0.5 \n",
    "\n",
    "        T = len(dftt)**0.5 * (Q_m1 - Q_pc) / σ\n",
    "        print('λ = ', λ, 'T stat =', T)\n",
    "        print(norm.sf(abs(T))*2)\n",
    "        \n",
    "        tlist1.append(T)\n",
    "        tlist2.append(norm.sf(abs(T))*2)\n",
    "tabs_tstat.append(tlist1 + tlist2)\n",
    "res_iv_bl = res2\n",
    "\n",
    "# No interaction\n",
    "\n",
    "# OLS\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935)].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "\n",
    "depvar = np.log(dft[spread])\n",
    "\n",
    "exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "res2 = sm.OLS(depvar, exog).fit()\n",
    "\n",
    "elas = (res2.params[debt]) \n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "\n",
    "tabs_elas.append(['ni_ols',\n",
    "                  (-1/elas).mean(),\n",
    "                  (-1/elas).mean(),\n",
    "                  -elas.mean(), \n",
    "                  -elas.mean()])\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "res_ols_ni = res2\n",
    "\n",
    "# IV\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "dft = df[df[var].notna()].copy()\n",
    "\n",
    "var = 'vix_hat'\n",
    "varma = var + '_ma'\n",
    "dft[varma] = dft[var].rolling(4).mean()\n",
    "dft = dft[dft[varma].notna()].copy()\n",
    "\n",
    "var = 'msdluk_vol_hat'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat_ma', 'rgdp_grate']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "dft = dft[(dft[var].notna()) & (dft.date >= 1935) & (dft.DLdep_rat_new.notna())].copy()\n",
    "λ = 1\n",
    "spread = 'spread_S60_L40'\n",
    "debt = 'lprivatepar'\n",
    "\n",
    "XX = sm.add_constant(dft[['vix_hat']]).to_numpy()\n",
    "proj = XX @ np.linalg.inv(XX.T @ XX) @ XX.T\n",
    "dft[var] = dft[var] - (proj @ dft[var]).flatten()\n",
    "\n",
    "vb = var + '_bin'\n",
    "dft[vb] = (dft[var] >= np.quantile(dft[var], 0.5)) + 0\n",
    "Fnow = 0\n",
    "\n",
    "for xx in np.arange(0, 13):\n",
    "    for v in np.arange(4, 164, 4):\n",
    "        if 'rameynewsgdp_%i_%i'%(xx, v) in dft.columns:\n",
    "            res1 = IV2SLS(dft[debt], add_constant(dft[['rameynewsgdp_%i_%i'%(xx, v),\n",
    "                                                           'DLdep_rat_new', 'slope', vb, 'vix_hat']]),\n",
    "                          None, None).fit()\n",
    "            if res1.f_statistic.stat > Fnow:\n",
    "                Fnow = res1.f_statistic.stat\n",
    "                \n",
    "                depvar = np.log(dft[[spread]])\n",
    "\n",
    "                exog = add_constant(dft[['slope', vb, debt, 'vix_hat']])\n",
    "\n",
    "                instr = add_constant(dft[['slope', vb, 'rameynewsgdp_%i_%i'%(xx, v), 'vix_hat']])\n",
    "                instr['DLdep_rat_new'] = dft.DLdep_rat_new\n",
    "                res2 = smIV2SLS(depvar, exog, instr).fit()\n",
    "\n",
    "                startlag = xx\n",
    "                endlag = v\n",
    "\n",
    "elas = (res2.params[debt])\n",
    "\n",
    "tabs_elas.append(['ni_iv',\n",
    "                  (-1/elas).mean(),\n",
    "                  (-1/elas).mean(),\n",
    "                  -elas.mean(), \n",
    "                  -elas.mean()])\n",
    "\n",
    "print('mean elas:', (-1/elas).mean())\n",
    "print('mean markup:', -elas.mean())\n",
    "\n",
    "dft['elas'] = elas\n",
    "\n",
    "\n",
    "res_iv_ni = res2\n",
    "\n",
    "sc = summary_col([res_ols_bl, res_iv_bl, res_ols_ni, res_iv_ni], stars = True, float_format = '%.2f', \n",
    "                regressor_order = ['lprivatepar', vb, 'lppXvol', 'vix_hat', 'slope', 'const'])\n",
    "\n",
    "tabs1 = pd.DataFrame(tabs_elas).T\n",
    "tabs1.columns = tabs1.iloc[0]\n",
    "tabs1.drop(index = 0, inplace = True)\n",
    "tabs1 = tabs1.astype(float).round(2)\n",
    "\n",
    "tabs2 = pd.DataFrame(tabs_tstat).T\n",
    "tabs2.columns = tabs2.iloc[0]\n",
    "tabs2.drop(index = 0, inplace = True)\n",
    "tabs2 = tabs2.astype(float).round(2)\n",
    "tabs2 = tabs2.T\n",
    "\n",
    "n = int(tabs2.shape[1] / 2)\n",
    "tabs3 = np.empty((tabs2.shape[0] * 2, 1 + n), dtype = object)\n",
    "tabs2 = tabs2.reset_index()\n",
    "for i in range(tabs2.shape[0]):\n",
    "    tabs3[i*2] = tabs2.iloc[i, :n+1]\n",
    "    tabs3[i*2+1, 1:] = '(' + tabs2.iloc[i, n+1:].astype(str) + ')'\n",
    "\n",
    "tab = sc.tables[0].copy()\n",
    "tab.reset_index(inplace = True)\n",
    "tab.columns = ['VARIABLES', 'OLS', 'IV', 'OLS', 'IV']\n",
    "tab.iloc[:, 0] = ['Log(debt/gdp)', '', 'Foreign Volatility Dummy (FVD)', '',\n",
    "                  'FVD $\\\\times$ Log(debt/gdp)', '',\n",
    "                  'US VIX', '', 'Yield curve slope', '', 'Constant', '', 'Observations', 'R-squared']\n",
    "tab.iloc[-2, 1:] = [int(res_ols_bl.nobs), int(res_iv_bl.nobs), int(res_ols_ni.nobs), int(res_iv_ni.nobs)]\n",
    "tab.iloc[-1, [2, 4]] = ['', '']\n",
    "\n",
    "tabs1 = tabs1.reset_index()\n",
    "tabs1.iloc[:, 0] = ['Demand elasticity, high foreign vol', 'Demand elasticity, low foreign vol',\n",
    "                    'Markup, high foreign vol', 'Markup, low foreign vol']\n",
    "tabs1.columns = tab.columns\n",
    "tab = pd.concat([tab, tabs1])\n",
    "tab.iloc[:, 1] = tab.iloc[:, 1].astype(str) + ' \\\\vspace{.5em}'\n",
    "\n",
    "table = texttable.Texttable()\n",
    "table.set_cols_align([\"l\", \"c\", \"c\", 'c', 'c'])\n",
    "table.set_cols_valign([\"t\", \"t\", \"t\", 't', 't'])\n",
    "table.set_cols_dtype(['t', 't', 't', 't', 't'])\n",
    "table.header(tab.columns)\n",
    "table.add_rows(tab.values.tolist(), header = False)\n",
    "\n",
    "string = latextable.draw_latex(table, use_booktabs = True,\n",
    "                                     multicolumn_header = [('', 1), ('(1)', 1), ('(2)', 1), ('(3)', 1), ('(4)', 1)])\n",
    "string = string.replace('\\n\\t\\t\\tObservations', '\\n\\t\\t\\t\\\\midrule\\n\\t\\t\\tObservations')\n",
    "tex_file = open('tab1_p.tex', 'w')\n",
    "tex_file.write(string)\n",
    "tex_file.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
